braintrust 0.0.61 → 0.0.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +1415 -5
- package/dist/cli.js +3681 -16515
- package/dist/framework.d.ts +13 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +3647 -16330
- package/dist/isomorph.d.ts +1 -2
- package/dist/logger.d.ts +119 -27
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/util.d.ts +1 -0
- package/package.json +3 -5
package/dist/browser.js
CHANGED
|
@@ -1,5 +1,1415 @@
|
|
|
1
|
-
function q(t,e){return function(){return t.apply(e,arguments)}}var{toString:kt}=Object.prototype,{getPrototypeOf:Ee}=Object,te=(t=>e=>{let r=kt.call(e);return t[r]||(t[r]=r.slice(8,-1).toLowerCase())})(Object.create(null)),O=t=>(t=t.toLowerCase(),e=>te(e)===t),re=t=>e=>typeof e===t,{isArray:L}=Array,$=re("undefined");function Ft(t){return t!==null&&!$(t)&&t.constructor!==null&&!$(t.constructor)&&_(t.constructor.isBuffer)&&t.constructor.isBuffer(t)}var Xe=O("ArrayBuffer");function jt(t){let e;return typeof ArrayBuffer<"u"&&ArrayBuffer.isView?e=ArrayBuffer.isView(t):e=t&&t.buffer&&Xe(t.buffer),e}var vt=re("string"),_=re("function"),Qe=re("number"),ne=t=>t!==null&&typeof t=="object",Bt=t=>t===!0||t===!1,ee=t=>{if(te(t)!=="object")return!1;let e=Ee(t);return(e===null||e===Object.prototype||Object.getPrototypeOf(e)===null)&&!(Symbol.toStringTag in t)&&!(Symbol.iterator in t)},zt=O("Date"),Mt=O("File"),Ht=O("Blob"),qt=O("FileList"),$t=t=>ne(t)&&_(t.pipe),Jt=t=>{let e;return t&&(typeof FormData=="function"&&t instanceof FormData||_(t.append)&&((e=te(t))==="formdata"||e==="object"&&_(t.toString)&&t.toString()==="[object FormData]"))},Vt=O("URLSearchParams"),Kt=t=>t.trim?t.trim():t.replace(/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,"");function J(t,e,{allOwnKeys:r=!1}={}){if(t===null||typeof t>"u")return;let n,i;if(typeof t!="object"&&(t=[t]),L(t))for(n=0,i=t.length;n<i;n++)e.call(null,t[n],n,t);else{let s=r?Object.getOwnPropertyNames(t):Object.keys(t),o=s.length,c;for(n=0;n<o;n++)c=s[n],e.call(null,t[c],c,t)}}function Ye(t,e){e=e.toLowerCase();let r=Object.keys(t),n=r.length,i;for(;n-- >0;)if(i=r[n],e===i.toLowerCase())return i;return null}var Ze=(()=>typeof globalThis<"u"?globalThis:typeof self<"u"?self:typeof window<"u"?window:global)(),et=t=>!$(t)&&t!==Ze;function we(){let{caseless:t}=et(this)&&this||{},e={},r=(n,i)=>{let s=t&&Ye(e,i)||i;ee(e[s])&&ee(n)?e[s]=we(e[s],n):ee(n)?e[s]=we({},n):L(n)?e[s]=n.slice():e[s]=n};for(let n=0,i=arguments.length;n<i;n++)arguments[n]&&J(arguments[n],r);return e}var Wt=(t,e,r,{allOwnKeys:n}={})=>(J(e,(i,s)=>{r&&_(i)?t[s]=q(i,r):t[s]=i},{allOwnKeys:n}),t),Gt=t=>(t.charCodeAt(0)===65279&&(t=t.slice(1)),t),Xt=(t,e,r,n)=>{t.prototype=Object.create(e.prototype,n),t.prototype.constructor=t,Object.defineProperty(t,"super",{value:e.prototype}),r&&Object.assign(t.prototype,r)},Qt=(t,e,r,n)=>{let i,s,o,c={};if(e=e||{},t==null)return e;do{for(i=Object.getOwnPropertyNames(t),s=i.length;s-- >0;)o=i[s],(!n||n(o,t,e))&&!c[o]&&(e[o]=t[o],c[o]=!0);t=r!==!1&&Ee(t)}while(t&&(!r||r(t,e))&&t!==Object.prototype);return e},Yt=(t,e,r)=>{t=String(t),(r===void 0||r>t.length)&&(r=t.length),r-=e.length;let n=t.indexOf(e,r);return n!==-1&&n===r},Zt=t=>{if(!t)return null;if(L(t))return t;let e=t.length;if(!Qe(e))return null;let r=new Array(e);for(;e-- >0;)r[e]=t[e];return r},er=(t=>e=>t&&e instanceof t)(typeof Uint8Array<"u"&&Ee(Uint8Array)),tr=(t,e)=>{let n=(t&&t[Symbol.iterator]).call(t),i;for(;(i=n.next())&&!i.done;){let s=i.value;e.call(t,s[0],s[1])}},rr=(t,e)=>{let r,n=[];for(;(r=t.exec(e))!==null;)n.push(r);return n},nr=O("HTMLFormElement"),ir=t=>t.toLowerCase().replace(/[-_\s]([a-z\d])(\w*)/g,function(r,n,i){return n.toUpperCase()+i}),We=(({hasOwnProperty:t})=>(e,r)=>t.call(e,r))(Object.prototype),sr=O("RegExp"),tt=(t,e)=>{let r=Object.getOwnPropertyDescriptors(t),n={};J(r,(i,s)=>{e(i,s,t)!==!1&&(n[s]=i)}),Object.defineProperties(t,n)},or=t=>{tt(t,(e,r)=>{if(_(t)&&["arguments","caller","callee"].indexOf(r)!==-1)return!1;let n=t[r];if(_(n)){if(e.enumerable=!1,"writable"in e){e.writable=!1;return}e.set||(e.set=()=>{throw Error("Can not rewrite read-only method '"+r+"'")})}})},ar=(t,e)=>{let r={},n=i=>{i.forEach(s=>{r[s]=!0})};return L(t)?n(t):n(String(t).split(e)),r},cr=()=>{},ur=(t,e)=>(t=+t,Number.isFinite(t)?t:e),be="abcdefghijklmnopqrstuvwxyz",Ge="0123456789",rt={DIGIT:Ge,ALPHA:be,ALPHA_DIGIT:be+be.toUpperCase()+Ge},lr=(t=16,e=rt.ALPHA_DIGIT)=>{let r="",{length:n}=e;for(;t--;)r+=e[Math.random()*n|0];return r};function dr(t){return!!(t&&_(t.append)&&t[Symbol.toStringTag]==="FormData"&&t[Symbol.iterator])}var pr=t=>{let e=new Array(10),r=(n,i)=>{if(ne(n)){if(e.indexOf(n)>=0)return;if(!("toJSON"in n)){e[i]=n;let s=L(n)?[]:{};return J(n,(o,c)=>{let d=r(o,i+1);!$(d)&&(s[c]=d)}),e[i]=void 0,s}}return n};return r(t,0)},fr=O("AsyncFunction"),mr=t=>t&&(ne(t)||_(t))&&_(t.then)&&_(t.catch),a={isArray:L,isArrayBuffer:Xe,isBuffer:Ft,isFormData:Jt,isArrayBufferView:jt,isString:vt,isNumber:Qe,isBoolean:Bt,isObject:ne,isPlainObject:ee,isUndefined:$,isDate:zt,isFile:Mt,isBlob:Ht,isRegExp:sr,isFunction:_,isStream:$t,isURLSearchParams:Vt,isTypedArray:er,isFileList:qt,forEach:J,merge:we,extend:Wt,trim:Kt,stripBOM:Gt,inherits:Xt,toFlatObject:Qt,kindOf:te,kindOfTest:O,endsWith:Yt,toArray:Zt,forEachEntry:tr,matchAll:rr,isHTMLForm:nr,hasOwnProperty:We,hasOwnProp:We,reduceDescriptors:tt,freezeMethods:or,toObjectSet:ar,toCamelCase:ir,noop:cr,toFiniteNumber:ur,findKey:Ye,global:Ze,isContextDefined:et,ALPHABET:rt,generateString:lr,isSpecCompliantForm:dr,toJSONObject:pr,isAsyncFn:fr,isThenable:mr};function D(t,e,r,n,i){Error.call(this),Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=new Error().stack,this.message=t,this.name="AxiosError",e&&(this.code=e),r&&(this.config=r),n&&(this.request=n),i&&(this.response=i)}a.inherits(D,Error,{toJSON:function(){return{message:this.message,name:this.name,description:this.description,number:this.number,fileName:this.fileName,lineNumber:this.lineNumber,columnNumber:this.columnNumber,stack:this.stack,config:a.toJSONObject(this.config),code:this.code,status:this.response&&this.response.status?this.response.status:null}}});var nt=D.prototype,it={};["ERR_BAD_OPTION_VALUE","ERR_BAD_OPTION","ECONNABORTED","ETIMEDOUT","ERR_NETWORK","ERR_FR_TOO_MANY_REDIRECTS","ERR_DEPRECATED","ERR_BAD_RESPONSE","ERR_BAD_REQUEST","ERR_CANCELED","ERR_NOT_SUPPORT","ERR_INVALID_URL"].forEach(t=>{it[t]={value:t}});Object.defineProperties(D,it);Object.defineProperty(nt,"isAxiosError",{value:!0});D.from=(t,e,r,n,i,s)=>{let o=Object.create(nt);return a.toFlatObject(t,o,function(d){return d!==Error.prototype},c=>c!=="isAxiosError"),D.call(o,t.message,e,r,n,i),o.cause=t,o.name=t.name,s&&Object.assign(o,s),o};var h=D;var ie=null;function Se(t){return a.isPlainObject(t)||a.isArray(t)}function ot(t){return a.endsWith(t,"[]")?t.slice(0,-2):t}function st(t,e,r){return t?t.concat(e).map(function(i,s){return i=ot(i),!r&&s?"["+i+"]":i}).join(r?".":""):e}function hr(t){return a.isArray(t)&&!t.some(Se)}var gr=a.toFlatObject(a,{},null,function(e){return/^is[A-Z]/.test(e)});function yr(t,e,r){if(!a.isObject(t))throw new TypeError("target must be an object");e=e||new(ie||FormData),r=a.toFlatObject(r,{metaTokens:!0,dots:!1,indexes:!1},!1,function(g,C){return!a.isUndefined(C[g])});let n=r.metaTokens,i=r.visitor||l,s=r.dots,o=r.indexes,d=(r.Blob||typeof Blob<"u"&&Blob)&&a.isSpecCompliantForm(e);if(!a.isFunction(i))throw new TypeError("visitor must be a function");function u(p){if(p===null)return"";if(a.isDate(p))return p.toISOString();if(!d&&a.isBlob(p))throw new h("Blob is not supported. Use a Buffer instead.");return a.isArrayBuffer(p)||a.isTypedArray(p)?d&&typeof Blob=="function"?new Blob([p]):Buffer.from(p):p}function l(p,g,C){let A=p;if(p&&!C&&typeof p=="object"){if(a.endsWith(g,"{}"))g=n?g:g.slice(0,-2),p=JSON.stringify(p);else if(a.isArray(p)&&hr(p)||(a.isFileList(p)||a.endsWith(g,"[]"))&&(A=a.toArray(p)))return g=ot(g),A.forEach(function(Z,Dt){!(a.isUndefined(Z)||Z===null)&&e.append(o===!0?st([g],Dt,s):o===null?g:g+"[]",u(Z))}),!1}return Se(p)?!0:(e.append(st(C,g,s),u(p)),!1)}let f=[],x=Object.assign(gr,{defaultVisitor:l,convertValue:u,isVisitable:Se});function y(p,g){if(!a.isUndefined(p)){if(f.indexOf(p)!==-1)throw Error("Circular reference detected in "+g.join("."));f.push(p),a.forEach(p,function(A,U){(!(a.isUndefined(A)||A===null)&&i.call(e,A,a.isString(U)?U.trim():U,g,x))===!0&&y(A,g?g.concat(U):[U])}),f.pop()}}if(!a.isObject(t))throw new TypeError("data must be an object");return y(t),e}var T=yr;function at(t){let e={"!":"%21","'":"%27","(":"%28",")":"%29","~":"%7E","%20":"+","%00":"\0"};return encodeURIComponent(t).replace(/[!'()~]|%20|%00/g,function(n){return e[n]})}function ct(t,e){this._pairs=[],t&&T(t,this,e)}var ut=ct.prototype;ut.append=function(e,r){this._pairs.push([e,r])};ut.toString=function(e){let r=e?function(n){return e.call(this,n,at)}:at;return this._pairs.map(function(i){return r(i[0])+"="+r(i[1])},"").join("&")};var se=ct;function xr(t){return encodeURIComponent(t).replace(/%3A/gi,":").replace(/%24/g,"$").replace(/%2C/gi,",").replace(/%20/g,"+").replace(/%5B/gi,"[").replace(/%5D/gi,"]")}function V(t,e,r){if(!e)return t;let n=r&&r.encode||xr,i=r&&r.serialize,s;if(i?s=i(e,r):s=a.isURLSearchParams(e)?e.toString():new se(e,r).toString(n),s){let o=t.indexOf("#");o!==-1&&(t=t.slice(0,o)),t+=(t.indexOf("?")===-1?"?":"&")+s}return t}var _e=class{constructor(){this.handlers=[]}use(e,r,n){return this.handlers.push({fulfilled:e,rejected:r,synchronous:n?n.synchronous:!1,runWhen:n?n.runWhen:null}),this.handlers.length-1}eject(e){this.handlers[e]&&(this.handlers[e]=null)}clear(){this.handlers&&(this.handlers=[])}forEach(e){a.forEach(this.handlers,function(n){n!==null&&e(n)})}},Re=_e;var oe={silentJSONParsing:!0,forcedJSONParsing:!0,clarifyTimeoutError:!1};var lt=typeof URLSearchParams<"u"?URLSearchParams:se;var dt=typeof FormData<"u"?FormData:null;var pt=typeof Blob<"u"?Blob:null;var br=(()=>{let t;return typeof navigator<"u"&&((t=navigator.product)==="ReactNative"||t==="NativeScript"||t==="NS")?!1:typeof window<"u"&&typeof document<"u"})(),wr=(()=>typeof WorkerGlobalScope<"u"&&self instanceof WorkerGlobalScope&&typeof self.importScripts=="function")(),E={isBrowser:!0,classes:{URLSearchParams:lt,FormData:dt,Blob:pt},isStandardBrowserEnv:br,isStandardBrowserWebWorkerEnv:wr,protocols:["http","https","file","blob","url","data"]};function Ae(t,e){return T(t,new E.classes.URLSearchParams,Object.assign({visitor:function(r,n,i,s){return E.isNode&&a.isBuffer(r)?(this.append(n,r.toString("base64")),!1):s.defaultVisitor.apply(this,arguments)}},e))}function Er(t){return a.matchAll(/\w+|\[(\w*)]/g,t).map(e=>e[0]==="[]"?"":e[1]||e[0])}function Sr(t){let e={},r=Object.keys(t),n,i=r.length,s;for(n=0;n<i;n++)s=r[n],e[s]=t[s];return e}function _r(t){function e(r,n,i,s){let o=r[s++],c=Number.isFinite(+o),d=s>=r.length;return o=!o&&a.isArray(i)?i.length:o,d?(a.hasOwnProp(i,o)?i[o]=[i[o],n]:i[o]=n,!c):((!i[o]||!a.isObject(i[o]))&&(i[o]=[]),e(r,n,i[o],s)&&a.isArray(i[o])&&(i[o]=Sr(i[o])),!c)}if(a.isFormData(t)&&a.isFunction(t.entries)){let r={};return a.forEachEntry(t,(n,i)=>{e(Er(n),i,r,0)}),r}return null}var ae=_r;var Rr={"Content-Type":void 0};function Ar(t,e,r){if(a.isString(t))try{return(e||JSON.parse)(t),a.trim(t)}catch(n){if(n.name!=="SyntaxError")throw n}return(r||JSON.stringify)(t)}var ce={transitional:oe,adapter:["xhr","http"],transformRequest:[function(e,r){let n=r.getContentType()||"",i=n.indexOf("application/json")>-1,s=a.isObject(e);if(s&&a.isHTMLForm(e)&&(e=new FormData(e)),a.isFormData(e))return i&&i?JSON.stringify(ae(e)):e;if(a.isArrayBuffer(e)||a.isBuffer(e)||a.isStream(e)||a.isFile(e)||a.isBlob(e))return e;if(a.isArrayBufferView(e))return e.buffer;if(a.isURLSearchParams(e))return r.setContentType("application/x-www-form-urlencoded;charset=utf-8",!1),e.toString();let c;if(s){if(n.indexOf("application/x-www-form-urlencoded")>-1)return Ae(e,this.formSerializer).toString();if((c=a.isFileList(e))||n.indexOf("multipart/form-data")>-1){let d=this.env&&this.env.FormData;return T(c?{"files[]":e}:e,d&&new d,this.formSerializer)}}return s||i?(r.setContentType("application/json",!1),Ar(e)):e}],transformResponse:[function(e){let r=this.transitional||ce.transitional,n=r&&r.forcedJSONParsing,i=this.responseType==="json";if(e&&a.isString(e)&&(n&&!this.responseType||i)){let o=!(r&&r.silentJSONParsing)&&i;try{return JSON.parse(e)}catch(c){if(o)throw c.name==="SyntaxError"?h.from(c,h.ERR_BAD_RESPONSE,this,null,this.response):c}}return e}],timeout:0,xsrfCookieName:"XSRF-TOKEN",xsrfHeaderName:"X-XSRF-TOKEN",maxContentLength:-1,maxBodyLength:-1,env:{FormData:E.classes.FormData,Blob:E.classes.Blob},validateStatus:function(e){return e>=200&&e<300},headers:{common:{Accept:"application/json, text/plain, */*"}}};a.forEach(["delete","get","head"],function(e){ce.headers[e]={}});a.forEach(["post","put","patch"],function(e){ce.headers[e]=a.merge(Rr)});var k=ce;var Or=a.toObjectSet(["age","authorization","content-length","content-type","etag","expires","from","host","if-modified-since","if-unmodified-since","last-modified","location","max-forwards","proxy-authorization","referer","retry-after","user-agent"]),ft=t=>{let e={},r,n,i;return t&&t.split(`
|
|
2
|
-
|
|
3
|
-
`)}get[Symbol.toStringTag](){return"AxiosHeaders"}static from(e){return e instanceof this?e:new this(e)}static concat(e,...r){let n=new this(e);return r.forEach(i=>n.set(i)),n}static accessor(e){let n=(this[mt]=this[mt]={accessors:{}}).accessors,i=this.prototype;function s(o){let c=K(o);n[c]||(Pr(i,o),n[c]=!0)}return a.isArray(e)?e.forEach(s):s(e),this}};F.accessor(["Content-Type","Content-Length","Accept","Accept-Encoding","User-Agent","Authorization"]);a.freezeMethods(F.prototype);a.freezeMethods(F);var S=F;function W(t,e){let r=this||k,n=e||r,i=S.from(n.headers),s=n.data;return a.forEach(t,function(c){s=c.call(r,s,i.normalize(),e?e.status:void 0)}),i.normalize(),s}function G(t){return!!(t&&t.__CANCEL__)}function ht(t,e,r){h.call(this,t??"canceled",h.ERR_CANCELED,e,r),this.name="CanceledError"}a.inherits(ht,h,{__CANCEL__:!0});var P=ht;function Ce(t,e,r){let n=r.config.validateStatus;!r.status||!n||n(r.status)?t(r):e(new h("Request failed with status code "+r.status,[h.ERR_BAD_REQUEST,h.ERR_BAD_RESPONSE][Math.floor(r.status/100)-4],r.config,r.request,r))}var gt=E.isStandardBrowserEnv?function(){return{write:function(r,n,i,s,o,c){let d=[];d.push(r+"="+encodeURIComponent(n)),a.isNumber(i)&&d.push("expires="+new Date(i).toGMTString()),a.isString(s)&&d.push("path="+s),a.isString(o)&&d.push("domain="+o),c===!0&&d.push("secure"),document.cookie=d.join("; ")},read:function(r){let n=document.cookie.match(new RegExp("(^|;\\s*)("+r+")=([^;]*)"));return n?decodeURIComponent(n[3]):null},remove:function(r){this.write(r,"",Date.now()-864e5)}}}():function(){return{write:function(){},read:function(){return null},remove:function(){}}}();function Ne(t){return/^([a-z][a-z\d+\-.]*:)?\/\//i.test(t)}function Te(t,e){return e?t.replace(/\/+$/,"")+"/"+e.replace(/^\/+/,""):t}function X(t,e){return t&&!Ne(e)?Te(t,e):e}var yt=E.isStandardBrowserEnv?function(){let e=/(msie|trident)/i.test(navigator.userAgent),r=document.createElement("a"),n;function i(s){let o=s;return e&&(r.setAttribute("href",o),o=r.href),r.setAttribute("href",o),{href:r.href,protocol:r.protocol?r.protocol.replace(/:$/,""):"",host:r.host,search:r.search?r.search.replace(/^\?/,""):"",hash:r.hash?r.hash.replace(/^#/,""):"",hostname:r.hostname,port:r.port,pathname:r.pathname.charAt(0)==="/"?r.pathname:"/"+r.pathname}}return n=i(window.location.href),function(o){let c=a.isString(o)?i(o):o;return c.protocol===n.protocol&&c.host===n.host}}():function(){return function(){return!0}}();function Pe(t){let e=/^([-+\w]{1,25})(:?\/\/|:)/.exec(t);return e&&e[1]||""}function Ir(t,e){t=t||10;let r=new Array(t),n=new Array(t),i=0,s=0,o;return e=e!==void 0?e:1e3,function(d){let u=Date.now(),l=n[s];o||(o=u),r[i]=d,n[i]=u;let f=s,x=0;for(;f!==i;)x+=r[f++],f=f%t;if(i=(i+1)%t,i===s&&(s=(s+1)%t),u-o<e)return;let y=l&&u-l;return y?Math.round(x*1e3/y):void 0}}var xt=Ir;function bt(t,e){let r=0,n=xt(50,250);return i=>{let s=i.loaded,o=i.lengthComputable?i.total:void 0,c=s-r,d=n(c),u=s<=o;r=s;let l={loaded:s,total:o,progress:o?s/o:void 0,bytes:c,rate:d||void 0,estimated:d&&o&&u?(o-s)/d:void 0,event:i};l[e?"download":"upload"]=!0,t(l)}}var Ur=typeof XMLHttpRequest<"u",wt=Ur&&function(t){return new Promise(function(r,n){let i=t.data,s=S.from(t.headers).normalize(),o=t.responseType,c;function d(){t.cancelToken&&t.cancelToken.unsubscribe(c),t.signal&&t.signal.removeEventListener("abort",c)}a.isFormData(i)&&(E.isStandardBrowserEnv||E.isStandardBrowserWebWorkerEnv?s.setContentType(!1):s.setContentType("multipart/form-data;",!1));let u=new XMLHttpRequest;if(t.auth){let y=t.auth.username||"",p=t.auth.password?unescape(encodeURIComponent(t.auth.password)):"";s.set("Authorization","Basic "+btoa(y+":"+p))}let l=X(t.baseURL,t.url);u.open(t.method.toUpperCase(),V(l,t.params,t.paramsSerializer),!0),u.timeout=t.timeout;function f(){if(!u)return;let y=S.from("getAllResponseHeaders"in u&&u.getAllResponseHeaders()),g={data:!o||o==="text"||o==="json"?u.responseText:u.response,status:u.status,statusText:u.statusText,headers:y,config:t,request:u};Ce(function(A){r(A),d()},function(A){n(A),d()},g),u=null}if("onloadend"in u?u.onloadend=f:u.onreadystatechange=function(){!u||u.readyState!==4||u.status===0&&!(u.responseURL&&u.responseURL.indexOf("file:")===0)||setTimeout(f)},u.onabort=function(){u&&(n(new h("Request aborted",h.ECONNABORTED,t,u)),u=null)},u.onerror=function(){n(new h("Network Error",h.ERR_NETWORK,t,u)),u=null},u.ontimeout=function(){let p=t.timeout?"timeout of "+t.timeout+"ms exceeded":"timeout exceeded",g=t.transitional||oe;t.timeoutErrorMessage&&(p=t.timeoutErrorMessage),n(new h(p,g.clarifyTimeoutError?h.ETIMEDOUT:h.ECONNABORTED,t,u)),u=null},E.isStandardBrowserEnv){let y=(t.withCredentials||yt(l))&&t.xsrfCookieName&>.read(t.xsrfCookieName);y&&s.set(t.xsrfHeaderName,y)}i===void 0&&s.setContentType(null),"setRequestHeader"in u&&a.forEach(s.toJSON(),function(p,g){u.setRequestHeader(g,p)}),a.isUndefined(t.withCredentials)||(u.withCredentials=!!t.withCredentials),o&&o!=="json"&&(u.responseType=t.responseType),typeof t.onDownloadProgress=="function"&&u.addEventListener("progress",bt(t.onDownloadProgress,!0)),typeof t.onUploadProgress=="function"&&u.upload&&u.upload.addEventListener("progress",bt(t.onUploadProgress)),(t.cancelToken||t.signal)&&(c=y=>{u&&(n(!y||y.type?new P(null,t,u):y),u.abort(),u=null)},t.cancelToken&&t.cancelToken.subscribe(c),t.signal&&(t.signal.aborted?c():t.signal.addEventListener("abort",c)));let x=Pe(l);if(x&&E.protocols.indexOf(x)===-1){n(new h("Unsupported protocol "+x+":",h.ERR_BAD_REQUEST,t));return}u.send(i||null)})};var le={http:ie,xhr:wt};a.forEach(le,(t,e)=>{if(t){try{Object.defineProperty(t,"name",{value:e})}catch{}Object.defineProperty(t,"adapterName",{value:e})}});var Et={getAdapter:t=>{t=a.isArray(t)?t:[t];let{length:e}=t,r,n;for(let i=0;i<e&&(r=t[i],!(n=a.isString(r)?le[r.toLowerCase()]:r));i++);if(!n)throw n===!1?new h(`Adapter ${r} is not supported by the environment`,"ERR_NOT_SUPPORT"):new Error(a.hasOwnProp(le,r)?`Adapter '${r}' is not available in the build`:`Unknown adapter '${r}'`);if(!a.isFunction(n))throw new TypeError("adapter is not a function");return n},adapters:le};function Ie(t){if(t.cancelToken&&t.cancelToken.throwIfRequested(),t.signal&&t.signal.aborted)throw new P(null,t)}function de(t){return Ie(t),t.headers=S.from(t.headers),t.data=W.call(t,t.transformRequest),["post","put","patch"].indexOf(t.method)!==-1&&t.headers.setContentType("application/x-www-form-urlencoded",!1),Et.getAdapter(t.adapter||k.adapter)(t).then(function(n){return Ie(t),n.data=W.call(t,t.transformResponse,n),n.headers=S.from(n.headers),n},function(n){return G(n)||(Ie(t),n&&n.response&&(n.response.data=W.call(t,t.transformResponse,n.response),n.response.headers=S.from(n.response.headers))),Promise.reject(n)})}var St=t=>t instanceof S?t.toJSON():t;function N(t,e){e=e||{};let r={};function n(u,l,f){return a.isPlainObject(u)&&a.isPlainObject(l)?a.merge.call({caseless:f},u,l):a.isPlainObject(l)?a.merge({},l):a.isArray(l)?l.slice():l}function i(u,l,f){if(a.isUndefined(l)){if(!a.isUndefined(u))return n(void 0,u,f)}else return n(u,l,f)}function s(u,l){if(!a.isUndefined(l))return n(void 0,l)}function o(u,l){if(a.isUndefined(l)){if(!a.isUndefined(u))return n(void 0,u)}else return n(void 0,l)}function c(u,l,f){if(f in e)return n(u,l);if(f in t)return n(void 0,u)}let d={url:s,method:s,data:s,baseURL:o,transformRequest:o,transformResponse:o,paramsSerializer:o,timeout:o,timeoutMessage:o,withCredentials:o,adapter:o,responseType:o,xsrfCookieName:o,xsrfHeaderName:o,onUploadProgress:o,onDownloadProgress:o,decompress:o,maxContentLength:o,maxBodyLength:o,beforeRedirect:o,transport:o,httpAgent:o,httpsAgent:o,cancelToken:o,socketPath:o,responseEncoding:o,validateStatus:c,headers:(u,l)=>i(St(u),St(l),!0)};return a.forEach(Object.keys(Object.assign({},t,e)),function(l){let f=d[l]||i,x=f(t[l],e[l],l);a.isUndefined(x)&&f!==c||(r[l]=x)}),r}var pe="1.4.0";var Ue={};["object","boolean","number","function","string","symbol"].forEach((t,e)=>{Ue[t]=function(n){return typeof n===t||"a"+(e<1?"n ":" ")+t}});var _t={};Ue.transitional=function(e,r,n){function i(s,o){return"[Axios v"+pe+"] Transitional option '"+s+"'"+o+(n?". "+n:"")}return(s,o,c)=>{if(e===!1)throw new h(i(o," has been removed"+(r?" in "+r:"")),h.ERR_DEPRECATED);return r&&!_t[o]&&(_t[o]=!0,console.warn(i(o," has been deprecated since v"+r+" and will be removed in the near future"))),e?e(s,o,c):!0}};function Lr(t,e,r){if(typeof t!="object")throw new h("options must be an object",h.ERR_BAD_OPTION_VALUE);let n=Object.keys(t),i=n.length;for(;i-- >0;){let s=n[i],o=e[s];if(o){let c=t[s],d=c===void 0||o(c,s,t);if(d!==!0)throw new h("option "+s+" must be "+d,h.ERR_BAD_OPTION_VALUE);continue}if(r!==!0)throw new h("Unknown option "+s,h.ERR_BAD_OPTION)}}var fe={assertOptions:Lr,validators:Ue};var I=fe.validators,j=class{constructor(e){this.defaults=e,this.interceptors={request:new Re,response:new Re}}request(e,r){typeof e=="string"?(r=r||{},r.url=e):r=e||{},r=N(this.defaults,r);let{transitional:n,paramsSerializer:i,headers:s}=r;n!==void 0&&fe.assertOptions(n,{silentJSONParsing:I.transitional(I.boolean),forcedJSONParsing:I.transitional(I.boolean),clarifyTimeoutError:I.transitional(I.boolean)},!1),i!=null&&(a.isFunction(i)?r.paramsSerializer={serialize:i}:fe.assertOptions(i,{encode:I.function,serialize:I.function},!0)),r.method=(r.method||this.defaults.method||"get").toLowerCase();let o;o=s&&a.merge(s.common,s[r.method]),o&&a.forEach(["delete","get","head","post","put","patch","common"],p=>{delete s[p]}),r.headers=S.concat(o,s);let c=[],d=!0;this.interceptors.request.forEach(function(g){typeof g.runWhen=="function"&&g.runWhen(r)===!1||(d=d&&g.synchronous,c.unshift(g.fulfilled,g.rejected))});let u=[];this.interceptors.response.forEach(function(g){u.push(g.fulfilled,g.rejected)});let l,f=0,x;if(!d){let p=[de.bind(this),void 0];for(p.unshift.apply(p,c),p.push.apply(p,u),x=p.length,l=Promise.resolve(r);f<x;)l=l.then(p[f++],p[f++]);return l}x=c.length;let y=r;for(f=0;f<x;){let p=c[f++],g=c[f++];try{y=p(y)}catch(C){g.call(this,C);break}}try{l=de.call(this,y)}catch(p){return Promise.reject(p)}for(f=0,x=u.length;f<x;)l=l.then(u[f++],u[f++]);return l}getUri(e){e=N(this.defaults,e);let r=X(e.baseURL,e.url);return V(r,e.params,e.paramsSerializer)}};a.forEach(["delete","get","head","options"],function(e){j.prototype[e]=function(r,n){return this.request(N(n||{},{method:e,url:r,data:(n||{}).data}))}});a.forEach(["post","put","patch"],function(e){function r(n){return function(s,o,c){return this.request(N(c||{},{method:e,headers:n?{"Content-Type":"multipart/form-data"}:{},url:s,data:o}))}}j.prototype[e]=r(),j.prototype[e+"Form"]=r(!0)});var Q=j;var Le=class t{constructor(e){if(typeof e!="function")throw new TypeError("executor must be a function.");let r;this.promise=new Promise(function(s){r=s});let n=this;this.promise.then(i=>{if(!n._listeners)return;let s=n._listeners.length;for(;s-- >0;)n._listeners[s](i);n._listeners=null}),this.promise.then=i=>{let s,o=new Promise(c=>{n.subscribe(c),s=c}).then(i);return o.cancel=function(){n.unsubscribe(s)},o},e(function(s,o,c){n.reason||(n.reason=new P(s,o,c),r(n.reason))})}throwIfRequested(){if(this.reason)throw this.reason}subscribe(e){if(this.reason){e(this.reason);return}this._listeners?this._listeners.push(e):this._listeners=[e]}unsubscribe(e){if(!this._listeners)return;let r=this._listeners.indexOf(e);r!==-1&&this._listeners.splice(r,1)}static source(){let e;return{token:new t(function(i){e=i}),cancel:e}}},Rt=Le;function De(t){return function(r){return t.apply(null,r)}}function ke(t){return a.isObject(t)&&t.isAxiosError===!0}var Fe={Continue:100,SwitchingProtocols:101,Processing:102,EarlyHints:103,Ok:200,Created:201,Accepted:202,NonAuthoritativeInformation:203,NoContent:204,ResetContent:205,PartialContent:206,MultiStatus:207,AlreadyReported:208,ImUsed:226,MultipleChoices:300,MovedPermanently:301,Found:302,SeeOther:303,NotModified:304,UseProxy:305,Unused:306,TemporaryRedirect:307,PermanentRedirect:308,BadRequest:400,Unauthorized:401,PaymentRequired:402,Forbidden:403,NotFound:404,MethodNotAllowed:405,NotAcceptable:406,ProxyAuthenticationRequired:407,RequestTimeout:408,Conflict:409,Gone:410,LengthRequired:411,PreconditionFailed:412,PayloadTooLarge:413,UriTooLong:414,UnsupportedMediaType:415,RangeNotSatisfiable:416,ExpectationFailed:417,ImATeapot:418,MisdirectedRequest:421,UnprocessableEntity:422,Locked:423,FailedDependency:424,TooEarly:425,UpgradeRequired:426,PreconditionRequired:428,TooManyRequests:429,RequestHeaderFieldsTooLarge:431,UnavailableForLegalReasons:451,InternalServerError:500,NotImplemented:501,BadGateway:502,ServiceUnavailable:503,GatewayTimeout:504,HttpVersionNotSupported:505,VariantAlsoNegotiates:506,InsufficientStorage:507,LoopDetected:508,NotExtended:510,NetworkAuthenticationRequired:511};Object.entries(Fe).forEach(([t,e])=>{Fe[e]=t});var At=Fe;function Ot(t){let e=new Q(t),r=q(Q.prototype.request,e);return a.extend(r,Q.prototype,e,{allOwnKeys:!0}),a.extend(r,e,null,{allOwnKeys:!0}),r.create=function(i){return Ot(N(t,i))},r}var b=Ot(k);b.Axios=Q;b.CanceledError=P;b.CancelToken=Rt;b.isCancel=G;b.VERSION=pe;b.toFormData=T;b.AxiosError=h;b.Cancel=b.CanceledError;b.all=function(e){return Promise.all(e)};b.spread=De;b.isAxiosError=ke;b.mergeConfig=N;b.AxiosHeaders=S;b.formToJSON=t=>ae(a.isHTMLForm(t)?new FormData(t):t);b.HttpStatusCode=At;b.default=b;var v=b;var{Axios:Cs,AxiosError:Ct,CanceledError:Ns,isCancel:Ts,CancelToken:Ps,VERSION:Is,all:Us,Cancel:Ls,isAxiosError:Ds,spread:ks,toFormData:Fs,AxiosHeaders:js,HttpStatusCode:vs,formToJSON:Bs,mergeConfig:zs}=v;var me,Dr=new Uint8Array(16);function je(){if(!me&&(me=typeof crypto<"u"&&crypto.getRandomValues&&crypto.getRandomValues.bind(crypto),!me))throw new Error("crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported");return me(Dr)}var w=[];for(let t=0;t<256;++t)w.push((t+256).toString(16).slice(1));function Nt(t,e=0){return(w[t[e+0]]+w[t[e+1]]+w[t[e+2]]+w[t[e+3]]+"-"+w[t[e+4]]+w[t[e+5]]+"-"+w[t[e+6]]+w[t[e+7]]+"-"+w[t[e+8]]+w[t[e+9]]+"-"+w[t[e+10]]+w[t[e+11]]+w[t[e+12]]+w[t[e+13]]+w[t[e+14]]+w[t[e+15]]).toLowerCase()}var kr=typeof crypto<"u"&&crypto.randomUUID&&crypto.randomUUID.bind(crypto),ve={randomUUID:kr};function Fr(t,e,r){if(ve.randomUUID&&!e&&!t)return ve.randomUUID();t=t||{};let n=t.random||(t.rng||je)();if(n[6]=n[6]&15|64,n[8]=n[8]&63|128,e){r=r||0;for(let i=0;i<16;++i)e[r+i]=n[i];return e}return Nt(n)}var B=Fr;var Be=class{constructor(){}enterWith(e){}run(e,r){return r()}getStore(){}},jr={makeAxios:t=>v.create({...t}),getRepoStatus:async()=>{},getPastNAncestors:async()=>[],getEnv:t=>{},getCallerLocation:()=>{},newAsyncLocalStorage:()=>new Be},R=jr;var Tt="_xact_id",z="_is_merge";function M(t,e){let r=!0;try{let n=t();return n instanceof Promise?(r=!1,n.finally(e)):n}finally{r&&e()}}function ze(t,e){for(let[r,n]of Object.entries(e)){let i=t[r];i instanceof Object&&!Array.isArray(i)&&e instanceof Object&&!Array.isArray(n)?ze(i,n):t[r]=n}}function vr(t){let e=r=>t[r]??"";return e("experiment_id")+":"+e("dataset_id")+":"+e("prompt_session_id")+":"+e("id")}function Pt(t){let e=[],r=[];for(let i of t)i.id===void 0?e.push(i):r.push(i);let n={};for(let i of r){let s=vr(i),o=n[s];if(o!==void 0&&i[z]){let c=!o[z];ze(o,i),c&&delete o[z]}else n[s]=i}return e.push(...Object.values(n)),e}var He=class{constructor(){this.kind="span";this.id="",this.span_id="",this.root_span_id=""}log(e){}startSpan(e,r){return this}traced(e,r,n){return r(this)}end(e){return e?.endTime??ye()}close(e){return this.end(e)}},It=new He,he=class{constructor(){this.id=B(),this.currentExperiment=R.newAsyncLocalStorage(),this.currentSpan=R.newAsyncLocalStorage(),this.apiUrl=null,this.loginToken=null,this.orgId=null,this.orgName=null,this.logUrl=null,this.loggedIn=!1,this._apiConn=null,this._logConn=null,this._userInfo=null,globalThis.__inherited_braintrust_state=this}apiConn(){if(!this._apiConn){if(!this.apiUrl)throw new Error("Must initialize apiUrl before requesting apiConn");this._apiConn=new Y(this.apiUrl)}return this._apiConn}logConn(){if(!this._logConn){if(!this.logUrl)throw new Error("Must initialize logUrl before requesting logConn");this._logConn=new Y(this.logUrl)}return this._logConn}async userInfo(){return this._userInfo||(this._userInfo=await this.logConn().get_json("ping")),this._userInfo}setUserInfoIfNull(e){this._userInfo||(this._userInfo=e)}},m=globalThis.__inherited_braintrust_state||new he,fo=()=>m,qe=class{constructor(){this.unterminatedObjects=new Map,process.on("exit",()=>{this.warnUnterminated()})}addUnterminated(e,r){this.unterminatedObjects.set(e,r)}removeUnterminated(e){this.unterminatedObjects.delete(e)}warnUnterminated(){if(this.unterminatedObjects.size===0)return;let e="WARNING: Did not close the following braintrust objects. We recommend running `.close` on the listed objects, or by running them inside a callback so they are closed automatically:";this.unterminatedObjects.forEach((r,n)=>{let i=`
|
|
4
|
-
Object of type ${n?.constructor?.name}`;r&&(i+=` created at ${JSON.stringify(r)}`),e+=i}),console.warn(e)}},H=new qe,Y=class t{constructor(e){this.base_url=e,this.token=null,this.session=null,this._reset()}async ping(){try{let e=await this.get("ping");return m.setUserInfoIfNull(e.data),e.status===200}catch{return!1}}make_long_lived(){this._reset()}static sanitize_token(e){return e.trim()}set_token(e){e=t.sanitize_token(e),this.token=e,this._reset()}_reset(){let e={};this.token&&(e.Authorization=`Bearer ${this.token}`),this.session=R.makeAxios({headers:e})}async get(e,r=void 0){return await this.session.get($e(this.base_url,e),{params:r})}async post(e,r=void 0,n=void 0){return await this.session.post($e(this.base_url,e),r,n)}async get_json(e,r=void 0,n=0){let i=n+1;for(let s=0;s<i;s++)try{return(await this.get(`${e}`,r)).data}catch(o){if(s<i-1){console.log(`Retrying API request ${e} ${r} ${o.status} ${o.text}`);continue}throw o}}async post_json(e,r=void 0){return(await this.post(`${e}`,r,{headers:{"Content-Type":"application/json"}})).data}},Ut=class{constructor(e,r,n){this.name=e,this.id=r,this.org_id=n}},Br=6*1024*1024;function zr(t){return`[${t.join(",")}]`}var Mr=100,Me=3,ge=class{constructor(){this.items=[];this.active_flush=Promise.resolve([]);this.active_flush_resolved=!0}log(e){this.items.push(...e),this.active_flush_resolved&&(this.active_flush_resolved=!1,this.active_flush=this.flush_once())}async flush_once(e=Mr){this.active_flush_resolved=!1;let r=Pt(this.items||[]).reverse();this.items=[];let n=[];for(;;){let s=[],o=0;for(;s.length<e&&o<Br/2;){let c=null;if(r.length>0)c=r.pop();else break;let d=JSON.stringify(c);s.push(d),o+=d.length}if(s.length===0)break;n.push((async()=>{let c=zr(s);for(let d=0;d<Me;d++){let u=performance.now();try{return(await m.logConn().post_json("logs",c)).map(l=>l.id)}catch(l){let f=d+1===Me?"":" Retrying",x=(()=>l instanceof Ct&&l.response?`${l.response.status}: ${JSON.stringify(l.response.data)}`:`${l}`)();console.warn(`log request failed. Elapsed time: ${(performance.now()-u)/1e3} seconds. Payload size: ${c.length}. Error: ${x}.${f}`)}}return console.warn(`log request failed after ${Me} retries. Dropping batch`),[]})())}let i=await Promise.all(n);return this.items.length>0?this.active_flush=this.flush_once():this.active_flush_resolved=!0,i}async flush(){for(;await this.active_flush,!this.active_flush_resolved;);}};async function Hr(t,e={}){let{experiment:r,description:n,dataset:i,baseExperiment:s,isPublic:o,update:c,apiUrl:d,apiKey:u,orgName:l,disableCache:f}=e||{};return await Lt({orgName:l,disableCache:f,apiKey:u,apiUrl:d}),await Xr(t,{experimentName:r,description:n,dataset:i,update:c,baseExperiment:s,isPublic:o})}async function mo(t,e,r={}){let n=await Hr(t,r);return M(()=>r.setCurrent??!0?xe(n,()=>e(n)):e(n),()=>n.close())}async function qr(t,e={}){let{dataset:r,description:n,version:i,apiUrl:s,apiKey:o,orgName:c,disableCache:d}=e||{};return await Lt({orgName:c,disableCache:d,apiKey:o,apiUrl:s}),await Qr(t,{name:r,description:n,version:i})}async function ho(t,e,r={}){let n=await qr(t,r);return M(()=>e(n),()=>n.close())}async function Lt(t={}){let{apiUrl:e=R.getEnv("BRAINTRUST_API_URL")||"https://www.braintrustdata.com",apiKey:r=R.getEnv("BRAINTRUST_API_KEY"),orgName:n=void 0,disableCache:i=!1}=t||{},{forceLogin:s=!1}=t||{};if((e!=m.apiUrl||r!==void 0&&Y.sanitize_token(r)!=m.loginToken||n!==void 0&&n!=m.orgName)&&(s=!0),m.loggedIn&&!s)return;m=new he,m.apiUrl=e;let o=null,c=!1,d=null;if(r!==void 0){let l=(await v.post($e(m.apiUrl,"/api/apikey/login"),{token:r})).data;Kr(l.org_info,n),d=m.logConn(),d.set_token(r),c=await d.ping()}else throw new Error("Please specify an api key. Token based login is not yet implemented in the JS client.");if(!d)throw new Error("Conn should be set at this point (a bug)");c||await d.get("ping"),d.make_long_lived(),m.apiConn().set_token(r),m.loginToken=d.token,m.loggedIn=!0}function go(t){let e=m.currentExperiment.getStore();if(!e)throw new Error("Not initialized. Please call init() first");return e.log(t)}async function yo(t={}){let e=m.currentExperiment.getStore();if(!e)throw new Error("Not initialized. Please call init() first");return await e.summarize(t)}function $r(){return m.currentExperiment.getStore()}function Jr(){return m.currentSpan.getStore()}function Vr(t){let{name:e,...r}=t??{},n=(e??R.getCallerLocation()?.caller_functionname)||"root",i=Jr();if(!Object.is(i,It))return i.startSpan(n,r);let s=$r();return s?s.startSpan({name:n,...r}):It}function xo(t,e){let r=Vr(e);return M(()=>e?.setCurrent??!0?xe(r,()=>t(r)):t(r),()=>r.end())}function xe(t,e){if(t.kind==="experiment")return m.currentExperiment.run(t,e);if(t.kind==="span")return m.currentSpan.run(t,e);throw new Error(`Invalid object of type ${t.constructor.name}`)}function Kr(t,e){if(t.length===0)throw new Error("This user is not part of any organizations.");for(let r of t)if(e===void 0||r.name===e){m.orgId=r.id,m.orgName=r.name,m.logUrl=r.api_url;break}if(m.orgId===void 0)throw new Error(`Organization ${e} not found. Must be one of ${t.map(r=>r.name).join(", ")}`)}function $e(...t){return t.map(e=>e.replace(/^\//,"")).join("/")}function ye(){return new Date().getTime()/1e3}function Wr(t){if(t.scores)for(let[e,r]of Object.entries(t.scores)){if(typeof e!="string")throw new Error("score names must be strings");if(typeof r=="boolean"&&(r=r?1:0,t.scores[e]=r),typeof r!="number")throw new Error("score values must be numbers");if(r<0||r>1)throw new Error("score values must be between 0 and 1")}if(t.metadata){for(let e of Object.keys(t.metadata))if(typeof e!="string")throw new Error("metadata keys must be strings")}if(t.metrics){for(let e of Object.keys(t.metrics))if(typeof e!="string")throw new Error("metric keys must be strings");for(let e of["start","end","caller_functionname","caller_filename","caller_lineno"])if(e in t.metrics)throw new Error(`Key ${e} may not be specified in metrics`)}if("input"in t&&t.input&&"inputs"in t&&t.inputs)throw new Error("Only one of input or inputs (deprecated) can be specified. Prefer input.");if("inputs"in t){let{inputs:e,...r}=t;return{input:e,...r}}else return{...t}}function Gr(t,e){if("input"in t&&t.input&&"inputs"in t&&t.inputs||!("input"in t)&&!("inputs"in t))throw new Error("Exactly one of input or inputs (deprecated) must be specified. Prefer input.");if(!t.scores)throw new Error("scores must be specified");if(e&&t.datasetRecordId===void 0)throw new Error("datasetRecordId must be specified when using a dataset");if(!e&&t.datasetRecordId!==void 0)throw new Error("datasetRecordId cannot be specified when not using a dataset");return t}async function Xr(t,{experimentName:e,description:r,dataset:n,update:i,baseExperiment:s,isPublic:o}={experimentName:void 0,description:void 0,baseExperiment:void 0,isPublic:!1}){let c={project_name:t,org_id:m.orgId};e&&(c.experiment_name=e),r&&(c.description=r),i&&(c.update=i);let d=await R.getRepoStatus();d&&(c.repo_info=d),s?c.base_experiment=s:c.ancestor_commits=await R.getPastNAncestors(),n!==void 0&&(c.dataset_id=n.id,c.dataset_version=await n.version()),o!==void 0&&(c.public=o);let u=await m.apiConn().post_json("api/experiment/register",c),l=u.project,f=u.experiment,x=(await m.userInfo()).id;return new Je(l,f.id,f.name,x,n)}var Je=class{constructor(e,r,n,i,s){this.kind="experiment";this.finished=!1,this.project=e,this.id=r,this.name=n,this.user_id=i,this.dataset=s,this.logger=new ge,this.lastStartTime=ye(),H.addUnterminated(this,R.getCallerLocation())}log(e){this.checkNotFinished(),e=Gr(e,!!this.dataset);let r=this.startSpan({startTime:this.lastStartTime,event:e});return this.lastStartTime=r.end(),r.id}startSpan(e){this.checkNotFinished();let{name:r,...n}=e??{};return new Ve({experimentLogger:this.logger,name:r??"root",...n,rootExperiment:this})}traced(e,r){let{setCurrent:n,...i}=r??{},s=this.startSpan(i);return M(()=>n??!0?xe(s,()=>e(s)):e(s),()=>s.end())}async summarize(e={}){let{summarizeScores:r=!0,comparisonExperimentId:n=void 0}=e||{};await this.logger.flush();let i=`${m.apiUrl}/app/${encodeURIComponent(m.orgName)}/p/${encodeURIComponent(this.project.name)}`,s=`${i}/${encodeURIComponent(this.name)}`,o,c;if(r){if(n===void 0){let l=(await m.logConn().get("/crud/base_experiments",{id:this.id})).data;l.length>0&&(n=l[0].base_exp_id,c=l[0].base_exp_name)}n!==void 0&&(o=await m.logConn().get_json("/experiment-comparison",{experiment_id:this.id,base_experiment_id:n},3))}return{projectName:this.project.name,experimentName:this.name,projectUrl:i,experimentUrl:s,comparisonExperimentName:c,scores:o}}async close(){return this.checkNotFinished(),await this.logger.flush(),this.finished=!0,H.removeUnterminated(this),this.id}checkNotFinished(){if(this.finished)throw new Error("Cannot invoke method on finished experiment")}},Ve=class t{constructor(e){this.kind="span";this.finished=!1,this.experimentLogger=e.experimentLogger;let r=R.getCallerLocation();if(this.internalData={metrics:{start:e.startTime??ye(),...r},span_attributes:{...e.spanAttributes,name:e.name}},this.id=e.event?.id??B(),this.span_id=B(),"rootExperiment"in e)this.root_span_id=this.span_id,this._project_id=e.rootExperiment.project.id,this._experiment_id=e.rootExperiment.id,this.internalData=Object.assign(this.internalData,{user_id:e.rootExperiment.user_id,created:new Date().toISOString()});else if("parentSpan"in e)this.root_span_id=e.parentSpan.root_span_id,this._project_id=e.parentSpan._project_id,this._experiment_id=e.parentSpan._experiment_id,this.internalData.span_parents=[e.parentSpan.span_id];else throw new Error("Must provide either 'rootExperiment' or 'parentSpan'");this.isMerge=!1;let{id:n,...i}=e.event??{};this.log(i),this.isMerge=!0,H.addUnterminated(this,r)}log(e){this.checkNotFinished();let n={...Wr(e),...this.internalData,id:this.id,span_id:this.span_id,root_span_id:this.root_span_id,project_id:this._project_id,experiment_id:this._experiment_id,[z]:this.isMerge};this.internalData={},this.experimentLogger.log([n])}startSpan(e,r){return this.checkNotFinished(),new t({experimentLogger:this.experimentLogger,name:e,...r,parentSpan:this})}traced(e,r,n){let{setCurrent:i,...s}=n??{},o=this.startSpan(e,s);return M(()=>i??!0?xe(o,()=>r(o)):r(o),()=>o.end())}end(e){this.checkNotFinished();let r=e?.endTime??ye();return this.internalData={metrics:{end:r}},this.log({}),this.finished=!0,H.removeUnterminated(this),r}close(e){return this.end(e)}checkNotFinished(){if(this.finished)throw new Error("Cannot invoke method on finished span")}};async function Qr(t,{name:e,description:r,version:n}={}){let i={org_id:m.orgId,project_name:t,dataset_name:e,description:r},s=await m.apiConn().post_json("api/dataset/register",i),o=s.project,c=s.dataset,d=(await m.userInfo()).id;return new Ke(o,c.id,c.name,d,n)}var Ke=class{constructor(e,r,n,i,s){this._fetchedData=void 0;this.finished=!1,this.project=e,this.id=r,this.name=n,this.user_id=i,this.pinnedVersion=s,this.logger=new ge,H.addUnterminated(this,R.getCallerLocation())}insert({input:e,output:r,metadata:n,id:i}){if(this.checkNotFinished(),n!==void 0){for(let o of Object.keys(n))if(typeof o!="string")throw new Error("metadata keys must be strings")}let s={id:i||B(),inputs:e,output:r,project_id:this.project.id,dataset_id:this.id,user_id:this.user_id,created:new Date().toISOString(),metadata:n};return this.logger.log([s]),s.id}delete(e){this.checkNotFinished();let r=this.user_id,n={id:e,project_id:this.project.id,dataset_id:this.id,user_id:r,created:new Date().toISOString(),_object_delete:!0};return this.logger.log([n]),n.id}async summarize(e={}){this.checkNotFinished();let{summarizeData:r=!0}=e||{};await this.logger.flush();let n=`${m.apiUrl}/app/${encodeURIComponent(m.orgName)}/p/${encodeURIComponent(this.project.name)}`,i=`${n}/d/${encodeURIComponent(this.name)}`,s;return r&&(s=await m.logConn().get_json("dataset-summary",{dataset_id:this.id},3)),{projectName:this.project.name,datasetName:this.name,projectUrl:n,datasetUrl:i,dataSummary:s}}async*fetch(){this.checkNotFinished();let e=await this.fetchedData();for(let r of e)yield{id:r.id,input:r.input&&JSON.parse(r.input),output:r.input&&JSON.parse(r.output),metadata:r.metadata&&JSON.parse(r.metadata)};this.clearCache()}[Symbol.asyncIterator](){return this.checkNotFinished(),this.fetch()}async fetchedData(){if(this.checkNotFinished(),this._fetchedData===void 0){let r=await(await m.logConn().get("object/dataset",{id:this.id,fmt:"json",version:this.pinnedVersion})).data;this._fetchedData=r.split(`
|
|
5
|
-
|
|
1
|
+
// node_modules/uuid/dist/esm-browser/rng.js
|
|
2
|
+
var getRandomValues;
|
|
3
|
+
var rnds8 = new Uint8Array(16);
|
|
4
|
+
function rng() {
|
|
5
|
+
if (!getRandomValues) {
|
|
6
|
+
getRandomValues = typeof crypto !== "undefined" && crypto.getRandomValues && crypto.getRandomValues.bind(crypto);
|
|
7
|
+
if (!getRandomValues) {
|
|
8
|
+
throw new Error("crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported");
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
return getRandomValues(rnds8);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// node_modules/uuid/dist/esm-browser/stringify.js
|
|
15
|
+
var byteToHex = [];
|
|
16
|
+
for (let i = 0; i < 256; ++i) {
|
|
17
|
+
byteToHex.push((i + 256).toString(16).slice(1));
|
|
18
|
+
}
|
|
19
|
+
function unsafeStringify(arr, offset = 0) {
|
|
20
|
+
return (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// node_modules/uuid/dist/esm-browser/native.js
|
|
24
|
+
var randomUUID = typeof crypto !== "undefined" && crypto.randomUUID && crypto.randomUUID.bind(crypto);
|
|
25
|
+
var native_default = {
|
|
26
|
+
randomUUID
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
// node_modules/uuid/dist/esm-browser/v4.js
|
|
30
|
+
function v4(options, buf, offset) {
|
|
31
|
+
if (native_default.randomUUID && !buf && !options) {
|
|
32
|
+
return native_default.randomUUID();
|
|
33
|
+
}
|
|
34
|
+
options = options || {};
|
|
35
|
+
const rnds = options.random || (options.rng || rng)();
|
|
36
|
+
rnds[6] = rnds[6] & 15 | 64;
|
|
37
|
+
rnds[8] = rnds[8] & 63 | 128;
|
|
38
|
+
if (buf) {
|
|
39
|
+
offset = offset || 0;
|
|
40
|
+
for (let i = 0; i < 16; ++i) {
|
|
41
|
+
buf[offset + i] = rnds[i];
|
|
42
|
+
}
|
|
43
|
+
return buf;
|
|
44
|
+
}
|
|
45
|
+
return unsafeStringify(rnds);
|
|
46
|
+
}
|
|
47
|
+
var v4_default = v4;
|
|
48
|
+
|
|
49
|
+
// src/isomorph.ts
|
|
50
|
+
var DefaultAsyncLocalStorage = class {
|
|
51
|
+
constructor() {
|
|
52
|
+
}
|
|
53
|
+
enterWith(_) {
|
|
54
|
+
}
|
|
55
|
+
run(_, callback) {
|
|
56
|
+
return callback();
|
|
57
|
+
}
|
|
58
|
+
getStore() {
|
|
59
|
+
return void 0;
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
var iso = {
|
|
63
|
+
getRepoStatus: async () => void 0,
|
|
64
|
+
getPastNAncestors: async () => [],
|
|
65
|
+
getEnv: (_name) => void 0,
|
|
66
|
+
getCallerLocation: () => void 0,
|
|
67
|
+
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
68
|
+
processOn: (_0, _1) => {
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
var isomorph_default = iso;
|
|
72
|
+
|
|
73
|
+
// src/util.ts
|
|
74
|
+
var TRANSACTION_ID_FIELD = "_xact_id";
|
|
75
|
+
var IS_MERGE_FIELD = "_is_merge";
|
|
76
|
+
var GLOBAL_PROJECT = "Global";
|
|
77
|
+
function runFinally(f, finallyF) {
|
|
78
|
+
let runSyncCleanup = true;
|
|
79
|
+
try {
|
|
80
|
+
const ret = f();
|
|
81
|
+
if (ret instanceof Promise) {
|
|
82
|
+
runSyncCleanup = false;
|
|
83
|
+
return ret.finally(finallyF);
|
|
84
|
+
} else {
|
|
85
|
+
return ret;
|
|
86
|
+
}
|
|
87
|
+
} finally {
|
|
88
|
+
if (runSyncCleanup) {
|
|
89
|
+
finallyF();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function mergeDicts(mergeInto, mergeFrom) {
|
|
94
|
+
for (const [k, mergeFromV] of Object.entries(mergeFrom)) {
|
|
95
|
+
const mergeIntoV = mergeInto[k];
|
|
96
|
+
if (mergeIntoV instanceof Object && !Array.isArray(mergeIntoV) && mergeFrom instanceof Object && !Array.isArray(mergeFromV)) {
|
|
97
|
+
mergeDicts(mergeIntoV, mergeFromV);
|
|
98
|
+
} else {
|
|
99
|
+
mergeInto[k] = mergeFromV;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// src/merge_row_batch.ts
|
|
105
|
+
var DATA_OBJECT_KEYS = [
|
|
106
|
+
"org_id",
|
|
107
|
+
"project_id",
|
|
108
|
+
"experiment_id",
|
|
109
|
+
"dataset_id",
|
|
110
|
+
"prompt_session_id",
|
|
111
|
+
"log_id"
|
|
112
|
+
];
|
|
113
|
+
function generateUniqueRowKey(row) {
|
|
114
|
+
const coalesceEmpty = (field) => row[field] ?? "";
|
|
115
|
+
return DATA_OBJECT_KEYS.concat(["id"]).map(coalesceEmpty).join(":");
|
|
116
|
+
}
|
|
117
|
+
function mergeRowBatch(rows) {
|
|
118
|
+
const out = [];
|
|
119
|
+
const remainingRows = [];
|
|
120
|
+
for (const row of rows) {
|
|
121
|
+
if (row["id"] === void 0) {
|
|
122
|
+
out.push(row);
|
|
123
|
+
} else {
|
|
124
|
+
remainingRows.push(row);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const rowGroups = {};
|
|
128
|
+
for (const row of remainingRows) {
|
|
129
|
+
const key = generateUniqueRowKey(row);
|
|
130
|
+
const existingRow = rowGroups[key];
|
|
131
|
+
if (existingRow !== void 0 && row[IS_MERGE_FIELD]) {
|
|
132
|
+
const preserveNoMerge = !existingRow[IS_MERGE_FIELD];
|
|
133
|
+
mergeDicts(existingRow, row);
|
|
134
|
+
if (preserveNoMerge) {
|
|
135
|
+
delete existingRow[IS_MERGE_FIELD];
|
|
136
|
+
}
|
|
137
|
+
} else {
|
|
138
|
+
rowGroups[key] = row;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
out.push(...Object.values(rowGroups));
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// src/logger.ts
|
|
146
|
+
var NoopSpan = class {
|
|
147
|
+
constructor() {
|
|
148
|
+
this.kind = "span";
|
|
149
|
+
this.id = "";
|
|
150
|
+
this.span_id = "";
|
|
151
|
+
this.root_span_id = "";
|
|
152
|
+
}
|
|
153
|
+
log(_) {
|
|
154
|
+
}
|
|
155
|
+
startSpan(_0, _1) {
|
|
156
|
+
return this;
|
|
157
|
+
}
|
|
158
|
+
traced(_0, callback, _1) {
|
|
159
|
+
return callback(this);
|
|
160
|
+
}
|
|
161
|
+
end(args) {
|
|
162
|
+
return args?.endTime ?? getCurrentUnixTimestamp();
|
|
163
|
+
}
|
|
164
|
+
close(args) {
|
|
165
|
+
return this.end(args);
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
var noopSpan = new NoopSpan();
|
|
169
|
+
var BraintrustState = class {
|
|
170
|
+
constructor() {
|
|
171
|
+
this.id = v4_default();
|
|
172
|
+
this.currentExperiment = isomorph_default.newAsyncLocalStorage();
|
|
173
|
+
this.currentLogger = isomorph_default.newAsyncLocalStorage();
|
|
174
|
+
this.currentSpan = isomorph_default.newAsyncLocalStorage();
|
|
175
|
+
this.currentSpan.enterWith(noopSpan);
|
|
176
|
+
this.apiUrl = null;
|
|
177
|
+
this.loginToken = null;
|
|
178
|
+
this.orgId = null;
|
|
179
|
+
this.orgName = null;
|
|
180
|
+
this.logUrl = null;
|
|
181
|
+
this.loggedIn = false;
|
|
182
|
+
this._apiConn = null;
|
|
183
|
+
this._logConn = null;
|
|
184
|
+
this._userInfo = null;
|
|
185
|
+
globalThis.__inherited_braintrust_state = this;
|
|
186
|
+
}
|
|
187
|
+
apiConn() {
|
|
188
|
+
if (!this._apiConn) {
|
|
189
|
+
if (!this.apiUrl) {
|
|
190
|
+
throw new Error("Must initialize apiUrl before requesting apiConn");
|
|
191
|
+
}
|
|
192
|
+
this._apiConn = new HTTPConnection(this.apiUrl);
|
|
193
|
+
}
|
|
194
|
+
return this._apiConn;
|
|
195
|
+
}
|
|
196
|
+
logConn() {
|
|
197
|
+
if (!this._logConn) {
|
|
198
|
+
if (!this.logUrl) {
|
|
199
|
+
throw new Error("Must initialize logUrl before requesting logConn");
|
|
200
|
+
}
|
|
201
|
+
this._logConn = new HTTPConnection(this.logUrl);
|
|
202
|
+
}
|
|
203
|
+
return this._logConn;
|
|
204
|
+
}
|
|
205
|
+
async userInfo() {
|
|
206
|
+
if (!this._userInfo) {
|
|
207
|
+
this._userInfo = await this.logConn().get_json("ping");
|
|
208
|
+
}
|
|
209
|
+
return this._userInfo;
|
|
210
|
+
}
|
|
211
|
+
setUserInfoIfNull(info) {
|
|
212
|
+
if (!this._userInfo) {
|
|
213
|
+
this._userInfo = info;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
var _state = globalThis.__inherited_braintrust_state || new BraintrustState();
|
|
218
|
+
var _internalGetGlobalState = () => _state;
|
|
219
|
+
var UnterminatedObjectsHandler = class {
|
|
220
|
+
constructor() {
|
|
221
|
+
this.unterminatedObjects = /* @__PURE__ */ new Map();
|
|
222
|
+
isomorph_default.processOn("exit", () => {
|
|
223
|
+
this.warnUnterminated();
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
addUnterminated(obj, createdLocation) {
|
|
227
|
+
this.unterminatedObjects.set(obj, createdLocation);
|
|
228
|
+
}
|
|
229
|
+
removeUnterminated(obj) {
|
|
230
|
+
this.unterminatedObjects.delete(obj);
|
|
231
|
+
}
|
|
232
|
+
warnUnterminated() {
|
|
233
|
+
if (this.unterminatedObjects.size === 0) {
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
let warningMessage = "WARNING: Did not close the following braintrust objects. We recommend running `.close` on the listed objects, or by running them inside a callback so they are closed automatically:";
|
|
237
|
+
this.unterminatedObjects.forEach((createdLocation, obj) => {
|
|
238
|
+
let msg = `
|
|
239
|
+
Object of type ${obj?.constructor?.name}`;
|
|
240
|
+
if (createdLocation) {
|
|
241
|
+
msg += ` created at ${JSON.stringify(createdLocation)}`;
|
|
242
|
+
}
|
|
243
|
+
warningMessage += msg;
|
|
244
|
+
});
|
|
245
|
+
console.warn(warningMessage);
|
|
246
|
+
}
|
|
247
|
+
};
|
|
248
|
+
var unterminatedObjects = new UnterminatedObjectsHandler();
|
|
249
|
+
var FailedHTTPResponse = class extends Error {
|
|
250
|
+
constructor(status, text, data = null) {
|
|
251
|
+
super(`${status}: ${text}`);
|
|
252
|
+
this.status = status;
|
|
253
|
+
this.text = text;
|
|
254
|
+
this.data = data;
|
|
255
|
+
}
|
|
256
|
+
};
|
|
257
|
+
async function checkResponse(resp) {
|
|
258
|
+
if (resp.ok) {
|
|
259
|
+
return resp;
|
|
260
|
+
} else {
|
|
261
|
+
throw new FailedHTTPResponse(
|
|
262
|
+
resp.status,
|
|
263
|
+
resp.statusText,
|
|
264
|
+
await resp.text()
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
var HTTPConnection = class _HTTPConnection {
|
|
269
|
+
constructor(base_url) {
|
|
270
|
+
this.base_url = base_url;
|
|
271
|
+
this.token = null;
|
|
272
|
+
this.headers = {};
|
|
273
|
+
this._reset();
|
|
274
|
+
}
|
|
275
|
+
async ping() {
|
|
276
|
+
try {
|
|
277
|
+
const resp = await this.get("ping");
|
|
278
|
+
_state.setUserInfoIfNull(await resp.json());
|
|
279
|
+
return resp.status === 200;
|
|
280
|
+
} catch (e) {
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
make_long_lived() {
|
|
285
|
+
this._reset();
|
|
286
|
+
}
|
|
287
|
+
static sanitize_token(token) {
|
|
288
|
+
return token.trim();
|
|
289
|
+
}
|
|
290
|
+
set_token(token) {
|
|
291
|
+
token = _HTTPConnection.sanitize_token(token);
|
|
292
|
+
this.token = token;
|
|
293
|
+
this._reset();
|
|
294
|
+
}
|
|
295
|
+
// As far as I can tell, you cannot set the retry/backoff factor here
|
|
296
|
+
_reset() {
|
|
297
|
+
this.headers = {};
|
|
298
|
+
if (this.token) {
|
|
299
|
+
this.headers["Authorization"] = `Bearer ${this.token}`;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
async get(path, params = void 0) {
|
|
303
|
+
const url = new URL(_urljoin(this.base_url, path));
|
|
304
|
+
url.search = new URLSearchParams(
|
|
305
|
+
params ? Object.fromEntries(
|
|
306
|
+
Object.entries(params).filter(([_, v]) => v !== void 0)
|
|
307
|
+
) : {}
|
|
308
|
+
).toString();
|
|
309
|
+
return await checkResponse(
|
|
310
|
+
await fetch(url, {
|
|
311
|
+
headers: this.headers,
|
|
312
|
+
keepalive: true
|
|
313
|
+
})
|
|
314
|
+
);
|
|
315
|
+
}
|
|
316
|
+
async post(path, params, config) {
|
|
317
|
+
const { headers, ...rest } = config || {};
|
|
318
|
+
return await checkResponse(
|
|
319
|
+
await fetch(_urljoin(this.base_url, path), {
|
|
320
|
+
method: "POST",
|
|
321
|
+
headers: {
|
|
322
|
+
"Content-Type": "application/json",
|
|
323
|
+
...this.headers,
|
|
324
|
+
...headers
|
|
325
|
+
},
|
|
326
|
+
body: typeof params === "string" ? params : params ? JSON.stringify(params) : void 0,
|
|
327
|
+
keepalive: true,
|
|
328
|
+
...rest
|
|
329
|
+
})
|
|
330
|
+
);
|
|
331
|
+
}
|
|
332
|
+
async get_json(object_type, args = void 0, retries = 0) {
|
|
333
|
+
const tries = retries + 1;
|
|
334
|
+
for (let i = 0; i < tries; i++) {
|
|
335
|
+
try {
|
|
336
|
+
const resp = await this.get(`${object_type}`, args);
|
|
337
|
+
return await resp.json();
|
|
338
|
+
} catch (e) {
|
|
339
|
+
if (i < tries - 1) {
|
|
340
|
+
console.log(
|
|
341
|
+
`Retrying API request ${object_type} ${args} ${e.status} ${e.text}`
|
|
342
|
+
);
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
throw e;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
async post_json(object_type, args = void 0) {
|
|
350
|
+
const resp = await this.post(`${object_type}`, args, {
|
|
351
|
+
headers: { "Content-Type": "application/json" }
|
|
352
|
+
});
|
|
353
|
+
return await resp.json();
|
|
354
|
+
}
|
|
355
|
+
};
|
|
356
|
+
var Project = class {
|
|
357
|
+
constructor({ name, id }) {
|
|
358
|
+
this.name = name;
|
|
359
|
+
this.id = id;
|
|
360
|
+
}
|
|
361
|
+
async lazyInit() {
|
|
362
|
+
if (this.id === void 0) {
|
|
363
|
+
const response = await _state.apiConn().post_json("api/project/register", {
|
|
364
|
+
project_name: this.name || GLOBAL_PROJECT,
|
|
365
|
+
org_id: _state.orgId
|
|
366
|
+
});
|
|
367
|
+
this.id = response.project.id;
|
|
368
|
+
this.name = response.project.name;
|
|
369
|
+
} else if (this.name === void 0) {
|
|
370
|
+
const response = await _state.apiConn().get_json("api/project", {
|
|
371
|
+
id: this.id
|
|
372
|
+
});
|
|
373
|
+
this.name = response.name;
|
|
374
|
+
}
|
|
375
|
+
return { id: this.id, name: this.name };
|
|
376
|
+
}
|
|
377
|
+
};
|
|
378
|
+
var Logger = class {
|
|
379
|
+
constructor(lazyLogin, lazyProject, logOptions = {}) {
|
|
380
|
+
this.loggedIn = false;
|
|
381
|
+
// For type identification.
|
|
382
|
+
this.kind = "logger";
|
|
383
|
+
this._lazyLogin = lazyLogin;
|
|
384
|
+
this.lazyProject = lazyProject;
|
|
385
|
+
this.logOptions = logOptions;
|
|
386
|
+
this.bgLogger = new BackgroundLogger();
|
|
387
|
+
this.lastStartTime = getCurrentUnixTimestamp();
|
|
388
|
+
}
|
|
389
|
+
async lazyLogin() {
|
|
390
|
+
if (!this.loggedIn) {
|
|
391
|
+
await this._lazyLogin();
|
|
392
|
+
this.lastStartTime = getCurrentUnixTimestamp();
|
|
393
|
+
this.loggedIn = true;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
|
|
398
|
+
*
|
|
399
|
+
* @param event The event to log.
|
|
400
|
+
* @param event.input: The arguments that uniquely define a user input (an arbitrary, JSON serializable object).
|
|
401
|
+
* @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
|
|
402
|
+
* @param event.expected: The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
403
|
+
* @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs.
|
|
404
|
+
* @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
|
|
405
|
+
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically and should not be specified: "start", "end".
|
|
406
|
+
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
407
|
+
* :returns: The `id` of the logged event.
|
|
408
|
+
*/
|
|
409
|
+
async log(event) {
|
|
410
|
+
const span = await this.startSpan({ startTime: this.lastStartTime, event });
|
|
411
|
+
this.lastStartTime = span.end();
|
|
412
|
+
if (!this.logOptions.asyncFlush) {
|
|
413
|
+
await this.flush();
|
|
414
|
+
}
|
|
415
|
+
return span.id;
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Create a new toplevel span. The name parameter is optional and defaults to "root".
|
|
419
|
+
*
|
|
420
|
+
* See `Span.startSpan` for full details.
|
|
421
|
+
*/
|
|
422
|
+
async startSpan(args) {
|
|
423
|
+
await this.lazyLogin();
|
|
424
|
+
const project = await this.lazyProject.lazyInit();
|
|
425
|
+
const { name, ...argsRest } = args ?? {};
|
|
426
|
+
return new SpanImpl({
|
|
427
|
+
bgLogger: this.bgLogger,
|
|
428
|
+
name: name ?? "root",
|
|
429
|
+
...argsRest,
|
|
430
|
+
rootProject: project
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* Wrapper over `Logger.startSpan`, which passes the initialized `Span` it to the given callback and ends it afterwards. See `Span.traced` for full details.
|
|
435
|
+
*/
|
|
436
|
+
async traced(callback, args) {
|
|
437
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
438
|
+
const span = await this.startSpan(argsRest);
|
|
439
|
+
try {
|
|
440
|
+
let ret = null;
|
|
441
|
+
return await (setCurrent ?? true ? withCurrent(span, () => callback(span)) : callback(span));
|
|
442
|
+
} finally {
|
|
443
|
+
span.end();
|
|
444
|
+
if (!this.logOptions.asyncFlush) {
|
|
445
|
+
await this.flush();
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
/*
|
|
450
|
+
* Flush any pending logs to the server.
|
|
451
|
+
*/
|
|
452
|
+
async flush() {
|
|
453
|
+
return await this.bgLogger.flush();
|
|
454
|
+
}
|
|
455
|
+
};
|
|
456
|
+
var MaxRequestSize = 6 * 1024 * 1024;
|
|
457
|
+
function constructJsonArray(items) {
|
|
458
|
+
return `[${items.join(",")}]`;
|
|
459
|
+
}
|
|
460
|
+
var DefaultBatchSize = 100;
|
|
461
|
+
var NumRetries = 3;
|
|
462
|
+
function now() {
|
|
463
|
+
return (/* @__PURE__ */ new Date()).getTime();
|
|
464
|
+
}
|
|
465
|
+
var BackgroundLogger = class {
|
|
466
|
+
constructor() {
|
|
467
|
+
this.items = [];
|
|
468
|
+
this.active_flush = Promise.resolve([]);
|
|
469
|
+
this.active_flush_resolved = true;
|
|
470
|
+
isomorph_default.processOn("beforeExit", async () => {
|
|
471
|
+
await this.flush();
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
log(items) {
|
|
475
|
+
this.items.push(...items);
|
|
476
|
+
if (this.active_flush_resolved) {
|
|
477
|
+
this.active_flush_resolved = false;
|
|
478
|
+
this.active_flush = this.flush_once();
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
async flush_once(batchSize = DefaultBatchSize) {
|
|
482
|
+
this.active_flush_resolved = false;
|
|
483
|
+
const allItems = mergeRowBatch(this.items || []).reverse();
|
|
484
|
+
this.items = [];
|
|
485
|
+
let postPromises = [];
|
|
486
|
+
while (true) {
|
|
487
|
+
const items = [];
|
|
488
|
+
let itemsLen = 0;
|
|
489
|
+
while (items.length < batchSize && itemsLen < MaxRequestSize / 2) {
|
|
490
|
+
let item = null;
|
|
491
|
+
if (allItems.length > 0) {
|
|
492
|
+
item = allItems.pop();
|
|
493
|
+
} else {
|
|
494
|
+
break;
|
|
495
|
+
}
|
|
496
|
+
const itemS = JSON.stringify(item);
|
|
497
|
+
items.push(itemS);
|
|
498
|
+
itemsLen += itemS.length;
|
|
499
|
+
}
|
|
500
|
+
if (items.length === 0) {
|
|
501
|
+
break;
|
|
502
|
+
}
|
|
503
|
+
postPromises.push(
|
|
504
|
+
(async () => {
|
|
505
|
+
const itemsS = constructJsonArray(items);
|
|
506
|
+
for (let i = 0; i < NumRetries; i++) {
|
|
507
|
+
const startTime = now();
|
|
508
|
+
try {
|
|
509
|
+
return (await _state.logConn().post_json("logs", itemsS)).map(
|
|
510
|
+
(res) => res.id
|
|
511
|
+
);
|
|
512
|
+
} catch (e) {
|
|
513
|
+
const retryingText = i + 1 === NumRetries ? "" : " Retrying";
|
|
514
|
+
const errMsg = (() => {
|
|
515
|
+
if (e instanceof FailedHTTPResponse) {
|
|
516
|
+
return `${e.status} (${e.text}): ${e.data}`;
|
|
517
|
+
} else {
|
|
518
|
+
return `${e}`;
|
|
519
|
+
}
|
|
520
|
+
})();
|
|
521
|
+
console.warn(
|
|
522
|
+
`log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${itemsS.length}. Error: ${errMsg}.${retryingText}`
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
console.warn(
|
|
527
|
+
`log request failed after ${NumRetries} retries. Dropping batch`
|
|
528
|
+
);
|
|
529
|
+
return [];
|
|
530
|
+
})()
|
|
531
|
+
);
|
|
532
|
+
}
|
|
533
|
+
let ret = await Promise.all(postPromises);
|
|
534
|
+
if (this.items.length > 0) {
|
|
535
|
+
this.active_flush = this.flush_once();
|
|
536
|
+
} else {
|
|
537
|
+
this.active_flush_resolved = true;
|
|
538
|
+
}
|
|
539
|
+
return ret;
|
|
540
|
+
}
|
|
541
|
+
async flush() {
|
|
542
|
+
while (true) {
|
|
543
|
+
await this.active_flush;
|
|
544
|
+
if (this.active_flush_resolved) {
|
|
545
|
+
break;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
};
|
|
550
|
+
async function init(project, options = {}) {
|
|
551
|
+
const {
|
|
552
|
+
experiment,
|
|
553
|
+
description,
|
|
554
|
+
dataset,
|
|
555
|
+
baseExperiment,
|
|
556
|
+
isPublic,
|
|
557
|
+
update,
|
|
558
|
+
apiUrl,
|
|
559
|
+
apiKey,
|
|
560
|
+
orgName,
|
|
561
|
+
disableCache
|
|
562
|
+
} = options || {};
|
|
563
|
+
await login({
|
|
564
|
+
orgName,
|
|
565
|
+
disableCache,
|
|
566
|
+
apiKey,
|
|
567
|
+
apiUrl
|
|
568
|
+
});
|
|
569
|
+
return await _initExperiment(project, {
|
|
570
|
+
experimentName: experiment,
|
|
571
|
+
description,
|
|
572
|
+
dataset,
|
|
573
|
+
update,
|
|
574
|
+
baseExperiment,
|
|
575
|
+
isPublic
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
async function withExperiment(project, callback, options = {}) {
|
|
579
|
+
const experiment = await init(project, options);
|
|
580
|
+
return runFinally(
|
|
581
|
+
() => {
|
|
582
|
+
if (options.setCurrent ?? true) {
|
|
583
|
+
return withCurrent(experiment, () => callback(experiment));
|
|
584
|
+
} else {
|
|
585
|
+
return callback(experiment);
|
|
586
|
+
}
|
|
587
|
+
},
|
|
588
|
+
() => experiment.close()
|
|
589
|
+
);
|
|
590
|
+
}
|
|
591
|
+
async function withLogger(callback, options = {}) {
|
|
592
|
+
const logger = initLogger(options);
|
|
593
|
+
return runFinally(
|
|
594
|
+
() => {
|
|
595
|
+
if (options.setCurrent ?? true) {
|
|
596
|
+
return withCurrent(logger, () => callback(logger));
|
|
597
|
+
} else {
|
|
598
|
+
return callback(logger);
|
|
599
|
+
}
|
|
600
|
+
},
|
|
601
|
+
() => logger.flush()
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
async function initDataset(project, options = {}) {
|
|
605
|
+
const {
|
|
606
|
+
dataset,
|
|
607
|
+
description,
|
|
608
|
+
version,
|
|
609
|
+
apiUrl,
|
|
610
|
+
apiKey,
|
|
611
|
+
orgName,
|
|
612
|
+
disableCache
|
|
613
|
+
} = options || {};
|
|
614
|
+
await login({
|
|
615
|
+
orgName,
|
|
616
|
+
disableCache,
|
|
617
|
+
apiKey,
|
|
618
|
+
apiUrl
|
|
619
|
+
});
|
|
620
|
+
return await _initDataset(project, {
|
|
621
|
+
name: dataset,
|
|
622
|
+
description,
|
|
623
|
+
version
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
async function withDataset(project, callback, options = {}) {
|
|
627
|
+
const dataset = await initDataset(project, options);
|
|
628
|
+
return runFinally(
|
|
629
|
+
() => callback(dataset),
|
|
630
|
+
() => dataset.close()
|
|
631
|
+
);
|
|
632
|
+
}
|
|
633
|
+
function initLogger(options = {}) {
|
|
634
|
+
const {
|
|
635
|
+
projectName,
|
|
636
|
+
projectId,
|
|
637
|
+
asyncFlush,
|
|
638
|
+
apiUrl,
|
|
639
|
+
apiKey,
|
|
640
|
+
orgName,
|
|
641
|
+
disableCache
|
|
642
|
+
} = options || {};
|
|
643
|
+
const lazyLogin = async () => {
|
|
644
|
+
await login({
|
|
645
|
+
orgName,
|
|
646
|
+
disableCache,
|
|
647
|
+
apiKey,
|
|
648
|
+
apiUrl
|
|
649
|
+
});
|
|
650
|
+
};
|
|
651
|
+
return new Logger(
|
|
652
|
+
lazyLogin,
|
|
653
|
+
new Project({
|
|
654
|
+
name: projectName,
|
|
655
|
+
id: projectId
|
|
656
|
+
}),
|
|
657
|
+
{
|
|
658
|
+
asyncFlush
|
|
659
|
+
}
|
|
660
|
+
);
|
|
661
|
+
}
|
|
662
|
+
async function login(options = {}) {
|
|
663
|
+
const {
|
|
664
|
+
apiUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") || "https://www.braintrustdata.com",
|
|
665
|
+
apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
|
|
666
|
+
orgName = void 0,
|
|
667
|
+
disableCache = false
|
|
668
|
+
} = options || {};
|
|
669
|
+
let { forceLogin = false } = options || {};
|
|
670
|
+
if (apiUrl != _state.apiUrl || apiKey !== void 0 && HTTPConnection.sanitize_token(apiKey) != _state.loginToken || orgName !== void 0 && orgName != _state.orgName) {
|
|
671
|
+
forceLogin = true;
|
|
672
|
+
}
|
|
673
|
+
if (_state.loggedIn && !forceLogin) {
|
|
674
|
+
return;
|
|
675
|
+
}
|
|
676
|
+
_state = new BraintrustState();
|
|
677
|
+
_state.apiUrl = apiUrl;
|
|
678
|
+
let login_key_info = null;
|
|
679
|
+
let ping_ok = false;
|
|
680
|
+
let conn = null;
|
|
681
|
+
if (apiKey !== void 0) {
|
|
682
|
+
const resp = await checkResponse(
|
|
683
|
+
await fetch(_urljoin(_state.apiUrl, `/api/apikey/login`), {
|
|
684
|
+
method: "POST",
|
|
685
|
+
headers: {
|
|
686
|
+
"Content-Type": "application/json"
|
|
687
|
+
},
|
|
688
|
+
body: JSON.stringify({
|
|
689
|
+
token: apiKey
|
|
690
|
+
})
|
|
691
|
+
})
|
|
692
|
+
);
|
|
693
|
+
const info = await resp.json();
|
|
694
|
+
_check_org_info(info.org_info, orgName);
|
|
695
|
+
conn = _state.logConn();
|
|
696
|
+
conn.set_token(apiKey);
|
|
697
|
+
ping_ok = await conn.ping();
|
|
698
|
+
} else {
|
|
699
|
+
throw new Error(
|
|
700
|
+
"Please specify an api key. Token based login is not yet implemented in the JS client."
|
|
701
|
+
);
|
|
702
|
+
}
|
|
703
|
+
if (!conn) {
|
|
704
|
+
throw new Error("Conn should be set at this point (a bug)");
|
|
705
|
+
}
|
|
706
|
+
if (!ping_ok) {
|
|
707
|
+
await conn.get("ping");
|
|
708
|
+
}
|
|
709
|
+
conn.make_long_lived();
|
|
710
|
+
_state.apiConn().set_token(apiKey);
|
|
711
|
+
_state.loginToken = conn.token;
|
|
712
|
+
_state.loggedIn = true;
|
|
713
|
+
}
|
|
714
|
+
function log(event) {
|
|
715
|
+
const currentExperiment2 = _state.currentExperiment.getStore();
|
|
716
|
+
if (!currentExperiment2) {
|
|
717
|
+
throw new Error("Not initialized. Please call init() first");
|
|
718
|
+
}
|
|
719
|
+
return currentExperiment2.log(event);
|
|
720
|
+
}
|
|
721
|
+
async function summarize(options = {}) {
|
|
722
|
+
const currentExperiment2 = _state.currentExperiment.getStore();
|
|
723
|
+
if (!currentExperiment2) {
|
|
724
|
+
throw new Error("Not initialized. Please call init() first");
|
|
725
|
+
}
|
|
726
|
+
return await currentExperiment2.summarize(options);
|
|
727
|
+
}
|
|
728
|
+
function currentExperiment() {
|
|
729
|
+
return _state.currentExperiment.getStore();
|
|
730
|
+
}
|
|
731
|
+
function currentLogger() {
|
|
732
|
+
return _state.currentLogger.getStore();
|
|
733
|
+
}
|
|
734
|
+
function currentSpan() {
|
|
735
|
+
return _state.currentSpan.getStore();
|
|
736
|
+
}
|
|
737
|
+
function startSpan(args) {
|
|
738
|
+
const { name: nameOpt, ...argsRest } = args ?? {};
|
|
739
|
+
const name = (nameOpt ?? isomorph_default.getCallerLocation()?.caller_functionname) || "root";
|
|
740
|
+
const parentSpan = currentSpan();
|
|
741
|
+
if (!Object.is(parentSpan, noopSpan)) {
|
|
742
|
+
return parentSpan.startSpan(name, argsRest);
|
|
743
|
+
}
|
|
744
|
+
const experiment = currentExperiment();
|
|
745
|
+
if (experiment) {
|
|
746
|
+
return experiment.startSpan({ name, ...argsRest });
|
|
747
|
+
}
|
|
748
|
+
const logger = currentLogger();
|
|
749
|
+
if (logger) {
|
|
750
|
+
throw new Error(
|
|
751
|
+
"Cannot start a span within a logger from startSpan(). Use logger.startSpan() instead."
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
return noopSpan;
|
|
755
|
+
}
|
|
756
|
+
function traced(callback, args) {
|
|
757
|
+
const span = startSpan(args);
|
|
758
|
+
return runFinally(
|
|
759
|
+
() => {
|
|
760
|
+
if (args?.setCurrent ?? true) {
|
|
761
|
+
return withCurrent(span, () => callback(span));
|
|
762
|
+
} else {
|
|
763
|
+
return callback(span);
|
|
764
|
+
}
|
|
765
|
+
},
|
|
766
|
+
() => span.end()
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
function withCurrent(object, callback) {
|
|
770
|
+
if (object.kind === "experiment") {
|
|
771
|
+
return _state.currentExperiment.run(object, callback);
|
|
772
|
+
} else if (object.kind === "logger") {
|
|
773
|
+
return _state.currentLogger.run(object, callback);
|
|
774
|
+
} else if (object.kind === "span") {
|
|
775
|
+
return _state.currentSpan.run(object, callback);
|
|
776
|
+
} else {
|
|
777
|
+
throw new Error(
|
|
778
|
+
`Invalid object of type ${object.constructor.name}`
|
|
779
|
+
);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
function _check_org_info(org_info, org_name) {
|
|
783
|
+
if (org_info.length === 0) {
|
|
784
|
+
throw new Error("This user is not part of any organizations.");
|
|
785
|
+
}
|
|
786
|
+
for (const org of org_info) {
|
|
787
|
+
if (org_name === void 0 || org.name === org_name) {
|
|
788
|
+
_state.orgId = org.id;
|
|
789
|
+
_state.orgName = org.name;
|
|
790
|
+
_state.logUrl = isomorph_default.getEnv("BRAINTRUST_LOG_URL") ?? org.api_url;
|
|
791
|
+
break;
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
if (_state.orgId === void 0) {
|
|
795
|
+
throw new Error(
|
|
796
|
+
`Organization ${org_name} not found. Must be one of ${org_info.map((x) => x.name).join(", ")}`
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
function _urljoin(...parts) {
|
|
801
|
+
return parts.map((x) => x.replace(/^\//, "")).join("/");
|
|
802
|
+
}
|
|
803
|
+
function getCurrentUnixTimestamp() {
|
|
804
|
+
return (/* @__PURE__ */ new Date()).getTime() / 1e3;
|
|
805
|
+
}
|
|
806
|
+
function validateAndSanitizeExperimentLogPartialArgs(event) {
|
|
807
|
+
if (event.scores) {
|
|
808
|
+
for (let [name, score] of Object.entries(event.scores)) {
|
|
809
|
+
if (typeof name !== "string") {
|
|
810
|
+
throw new Error("score names must be strings");
|
|
811
|
+
}
|
|
812
|
+
if (typeof score === "boolean") {
|
|
813
|
+
score = score ? 1 : 0;
|
|
814
|
+
event.scores[name] = score;
|
|
815
|
+
}
|
|
816
|
+
if (typeof score !== "number") {
|
|
817
|
+
throw new Error("score values must be numbers");
|
|
818
|
+
}
|
|
819
|
+
if (score < 0 || score > 1) {
|
|
820
|
+
throw new Error("score values must be between 0 and 1");
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
if (event.metadata) {
|
|
825
|
+
for (const key of Object.keys(event.metadata)) {
|
|
826
|
+
if (typeof key !== "string") {
|
|
827
|
+
throw new Error("metadata keys must be strings");
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
if (event.metrics) {
|
|
832
|
+
for (const key of Object.keys(event.metrics)) {
|
|
833
|
+
if (typeof key !== "string") {
|
|
834
|
+
throw new Error("metric keys must be strings");
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
for (const forbiddenKey of [
|
|
838
|
+
"start",
|
|
839
|
+
"end",
|
|
840
|
+
"caller_functionname",
|
|
841
|
+
"caller_filename",
|
|
842
|
+
"caller_lineno"
|
|
843
|
+
]) {
|
|
844
|
+
if (forbiddenKey in event.metrics) {
|
|
845
|
+
throw new Error(`Key ${forbiddenKey} may not be specified in metrics`);
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
if ("input" in event && event.input && "inputs" in event && event.inputs) {
|
|
850
|
+
throw new Error(
|
|
851
|
+
"Only one of input or inputs (deprecated) can be specified. Prefer input."
|
|
852
|
+
);
|
|
853
|
+
}
|
|
854
|
+
if ("inputs" in event) {
|
|
855
|
+
const { inputs, ...rest } = event;
|
|
856
|
+
return { input: inputs, ...rest };
|
|
857
|
+
} else {
|
|
858
|
+
return { ...event };
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
862
|
+
if ("input" in event && event.input && "inputs" in event && event.inputs || !("input" in event) && !("inputs" in event)) {
|
|
863
|
+
throw new Error(
|
|
864
|
+
"Exactly one of input or inputs (deprecated) must be specified. Prefer input."
|
|
865
|
+
);
|
|
866
|
+
}
|
|
867
|
+
if (!event.scores) {
|
|
868
|
+
throw new Error("scores must be specified");
|
|
869
|
+
}
|
|
870
|
+
if (hasDataset && event.datasetRecordId === void 0) {
|
|
871
|
+
throw new Error("datasetRecordId must be specified when using a dataset");
|
|
872
|
+
} else if (!hasDataset && event.datasetRecordId !== void 0) {
|
|
873
|
+
throw new Error(
|
|
874
|
+
"datasetRecordId cannot be specified when not using a dataset"
|
|
875
|
+
);
|
|
876
|
+
}
|
|
877
|
+
return event;
|
|
878
|
+
}
|
|
879
|
+
async function _initExperiment(projectName, {
|
|
880
|
+
experimentName,
|
|
881
|
+
description,
|
|
882
|
+
dataset,
|
|
883
|
+
update,
|
|
884
|
+
baseExperiment,
|
|
885
|
+
isPublic
|
|
886
|
+
} = {
|
|
887
|
+
experimentName: void 0,
|
|
888
|
+
description: void 0,
|
|
889
|
+
baseExperiment: void 0,
|
|
890
|
+
isPublic: false
|
|
891
|
+
}) {
|
|
892
|
+
const args = {
|
|
893
|
+
project_name: projectName,
|
|
894
|
+
org_id: _state.orgId
|
|
895
|
+
};
|
|
896
|
+
if (experimentName) {
|
|
897
|
+
args["experiment_name"] = experimentName;
|
|
898
|
+
}
|
|
899
|
+
if (description) {
|
|
900
|
+
args["description"] = description;
|
|
901
|
+
}
|
|
902
|
+
if (update) {
|
|
903
|
+
args["update"] = update;
|
|
904
|
+
}
|
|
905
|
+
const repoStatus = await isomorph_default.getRepoStatus();
|
|
906
|
+
if (repoStatus) {
|
|
907
|
+
args["repo_info"] = repoStatus;
|
|
908
|
+
}
|
|
909
|
+
if (baseExperiment) {
|
|
910
|
+
args["base_experiment"] = baseExperiment;
|
|
911
|
+
} else {
|
|
912
|
+
args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
|
|
913
|
+
}
|
|
914
|
+
if (dataset !== void 0) {
|
|
915
|
+
args["dataset_id"] = dataset.id;
|
|
916
|
+
args["dataset_version"] = await dataset.version();
|
|
917
|
+
}
|
|
918
|
+
if (isPublic !== void 0) {
|
|
919
|
+
args["public"] = isPublic;
|
|
920
|
+
}
|
|
921
|
+
const response = await _state.apiConn().post_json("api/experiment/register", args);
|
|
922
|
+
const project = response.project;
|
|
923
|
+
const experiment = response.experiment;
|
|
924
|
+
const user_id = (await _state.userInfo())["id"];
|
|
925
|
+
return new Experiment(
|
|
926
|
+
project,
|
|
927
|
+
experiment.id,
|
|
928
|
+
experiment.name,
|
|
929
|
+
user_id,
|
|
930
|
+
dataset
|
|
931
|
+
);
|
|
932
|
+
}
|
|
933
|
+
var Experiment = class {
|
|
934
|
+
constructor(project, id, name, user_id, dataset) {
|
|
935
|
+
// For type identification.
|
|
936
|
+
this.kind = "experiment";
|
|
937
|
+
this.finished = false;
|
|
938
|
+
this.project = project;
|
|
939
|
+
this.id = id;
|
|
940
|
+
this.name = name;
|
|
941
|
+
this.user_id = user_id;
|
|
942
|
+
this.dataset = dataset;
|
|
943
|
+
this.bgLogger = new BackgroundLogger();
|
|
944
|
+
this.lastStartTime = getCurrentUnixTimestamp();
|
|
945
|
+
unterminatedObjects.addUnterminated(this, isomorph_default.getCallerLocation());
|
|
946
|
+
}
|
|
947
|
+
/**
|
|
948
|
+
* Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
|
|
949
|
+
*
|
|
950
|
+
* @param event The event to log.
|
|
951
|
+
* @param event.input: The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical.
|
|
952
|
+
* @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
|
|
953
|
+
* @param event.expected: The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
954
|
+
* @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
|
|
955
|
+
* @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
|
|
956
|
+
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically and should not be specified: "start", "end".
|
|
957
|
+
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
958
|
+
* @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
|
|
959
|
+
* @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
|
|
960
|
+
* :returns: The `id` of the logged event.
|
|
961
|
+
*/
|
|
962
|
+
log(event) {
|
|
963
|
+
this.checkNotFinished();
|
|
964
|
+
event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
|
|
965
|
+
const span = this.startSpan({ startTime: this.lastStartTime, event });
|
|
966
|
+
this.lastStartTime = span.end();
|
|
967
|
+
return span.id;
|
|
968
|
+
}
|
|
969
|
+
/**
|
|
970
|
+
* Create a new toplevel span. The name parameter is optional and defaults to "root".
|
|
971
|
+
*
|
|
972
|
+
* See `Span.startSpan` for full details.
|
|
973
|
+
*/
|
|
974
|
+
startSpan(args) {
|
|
975
|
+
this.checkNotFinished();
|
|
976
|
+
const { name, ...argsRest } = args ?? {};
|
|
977
|
+
return new SpanImpl({
|
|
978
|
+
bgLogger: this.bgLogger,
|
|
979
|
+
name: name ?? "root",
|
|
980
|
+
...argsRest,
|
|
981
|
+
rootExperiment: this
|
|
982
|
+
});
|
|
983
|
+
}
|
|
984
|
+
/**
|
|
985
|
+
* Wrapper over `Experiment.startSpan`, which passes the initialized `Span` it to the given callback and ends it afterwards. See `Span.traced` for full details.
|
|
986
|
+
*/
|
|
987
|
+
traced(callback, args) {
|
|
988
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
989
|
+
const span = this.startSpan(argsRest);
|
|
990
|
+
return runFinally(
|
|
991
|
+
() => {
|
|
992
|
+
if (setCurrent ?? true) {
|
|
993
|
+
return withCurrent(span, () => callback(span));
|
|
994
|
+
} else {
|
|
995
|
+
return callback(span);
|
|
996
|
+
}
|
|
997
|
+
},
|
|
998
|
+
() => span.end()
|
|
999
|
+
);
|
|
1000
|
+
}
|
|
1001
|
+
/**
|
|
1002
|
+
* Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
1003
|
+
*
|
|
1004
|
+
* @param options Options for summarizing the experiment.
|
|
1005
|
+
* @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
|
|
1006
|
+
* @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
|
|
1007
|
+
* @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
1008
|
+
*/
|
|
1009
|
+
async summarize(options = {}) {
|
|
1010
|
+
let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
|
|
1011
|
+
await this.bgLogger.flush();
|
|
1012
|
+
const projectUrl = `${_state.apiUrl}/app/${encodeURIComponent(
|
|
1013
|
+
_state.orgName
|
|
1014
|
+
)}/p/${encodeURIComponent(this.project.name)}`;
|
|
1015
|
+
const experimentUrl = `${projectUrl}/${encodeURIComponent(this.name)}`;
|
|
1016
|
+
let scores = void 0;
|
|
1017
|
+
let comparisonExperimentName = void 0;
|
|
1018
|
+
if (summarizeScores) {
|
|
1019
|
+
if (comparisonExperimentId === void 0) {
|
|
1020
|
+
const conn = _state.logConn();
|
|
1021
|
+
const resp = await conn.get("/crud/base_experiments", {
|
|
1022
|
+
id: this.id
|
|
1023
|
+
});
|
|
1024
|
+
const base_experiments = await resp.json();
|
|
1025
|
+
if (base_experiments.length > 0) {
|
|
1026
|
+
comparisonExperimentId = base_experiments[0]["base_exp_id"];
|
|
1027
|
+
comparisonExperimentName = base_experiments[0]["base_exp_name"];
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
if (comparisonExperimentId !== void 0) {
|
|
1031
|
+
scores = await _state.logConn().get_json(
|
|
1032
|
+
"/experiment-comparison",
|
|
1033
|
+
{
|
|
1034
|
+
experiment_id: this.id,
|
|
1035
|
+
base_experiment_id: comparisonExperimentId
|
|
1036
|
+
},
|
|
1037
|
+
3
|
|
1038
|
+
);
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
return {
|
|
1042
|
+
projectName: this.project.name,
|
|
1043
|
+
experimentName: this.name,
|
|
1044
|
+
projectUrl,
|
|
1045
|
+
experimentUrl,
|
|
1046
|
+
comparisonExperimentName,
|
|
1047
|
+
scores
|
|
1048
|
+
};
|
|
1049
|
+
}
|
|
1050
|
+
/**
|
|
1051
|
+
* Finish the experiment and return its id. After calling close, you may not invoke any further methods on the experiment object.
|
|
1052
|
+
*
|
|
1053
|
+
* Will be invoked automatically if the experiment is wrapped in a callback passed to `braintrust.withExperiment`.
|
|
1054
|
+
*
|
|
1055
|
+
* @returns The experiment id.
|
|
1056
|
+
*/
|
|
1057
|
+
async close() {
|
|
1058
|
+
this.checkNotFinished();
|
|
1059
|
+
await this.bgLogger.flush();
|
|
1060
|
+
this.finished = true;
|
|
1061
|
+
unterminatedObjects.removeUnterminated(this);
|
|
1062
|
+
return this.id;
|
|
1063
|
+
}
|
|
1064
|
+
checkNotFinished() {
|
|
1065
|
+
if (this.finished) {
|
|
1066
|
+
throw new Error("Cannot invoke method on finished experiment");
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
};
|
|
1070
|
+
var SpanImpl = class _SpanImpl {
|
|
1071
|
+
// root_experiment should only be specified for a root span. parent_span
|
|
1072
|
+
// should only be specified for non-root spans.
|
|
1073
|
+
constructor(args) {
|
|
1074
|
+
this.kind = "span";
|
|
1075
|
+
this.finished = false;
|
|
1076
|
+
this.bgLogger = args.bgLogger;
|
|
1077
|
+
const callerLocation = isomorph_default.getCallerLocation();
|
|
1078
|
+
this.internalData = {
|
|
1079
|
+
metrics: {
|
|
1080
|
+
start: args.startTime ?? getCurrentUnixTimestamp(),
|
|
1081
|
+
...callerLocation
|
|
1082
|
+
},
|
|
1083
|
+
span_attributes: { ...args.spanAttributes, name: args.name }
|
|
1084
|
+
};
|
|
1085
|
+
this.id = args.event?.id ?? v4_default();
|
|
1086
|
+
this.span_id = v4_default();
|
|
1087
|
+
if ("rootExperiment" in args) {
|
|
1088
|
+
this.root_span_id = this.span_id;
|
|
1089
|
+
this._object_info = {
|
|
1090
|
+
project_id: args.rootExperiment.project.id,
|
|
1091
|
+
experiment_id: args.rootExperiment.id
|
|
1092
|
+
};
|
|
1093
|
+
this.internalData = Object.assign(this.internalData, {
|
|
1094
|
+
// TODO: Hopefully we can remove this.
|
|
1095
|
+
user_id: args.rootExperiment.user_id,
|
|
1096
|
+
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
1097
|
+
});
|
|
1098
|
+
} else if ("rootProject" in args) {
|
|
1099
|
+
this.root_span_id = this.span_id;
|
|
1100
|
+
this._object_info = {
|
|
1101
|
+
org_id: _state.orgId,
|
|
1102
|
+
project_id: args.rootProject.id,
|
|
1103
|
+
log_id: "g"
|
|
1104
|
+
};
|
|
1105
|
+
this.internalData = Object.assign(this.internalData, {
|
|
1106
|
+
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
1107
|
+
});
|
|
1108
|
+
} else if ("parentSpan" in args) {
|
|
1109
|
+
this.root_span_id = args.parentSpan.root_span_id;
|
|
1110
|
+
this._object_info = args.parentSpan._object_info;
|
|
1111
|
+
this.internalData.span_parents = [args.parentSpan.span_id];
|
|
1112
|
+
} else {
|
|
1113
|
+
throw new Error("Must provide either 'rootExperiment' or 'parentSpan'");
|
|
1114
|
+
}
|
|
1115
|
+
this.isMerge = false;
|
|
1116
|
+
const { id, ...eventRest } = args.event ?? {};
|
|
1117
|
+
this.log(eventRest);
|
|
1118
|
+
this.isMerge = true;
|
|
1119
|
+
unterminatedObjects.addUnterminated(this, callerLocation);
|
|
1120
|
+
}
|
|
1121
|
+
log(event) {
|
|
1122
|
+
this.checkNotFinished();
|
|
1123
|
+
const sanitized = validateAndSanitizeExperimentLogPartialArgs(event);
|
|
1124
|
+
const record = {
|
|
1125
|
+
...sanitized,
|
|
1126
|
+
...this.internalData,
|
|
1127
|
+
id: this.id,
|
|
1128
|
+
span_id: this.span_id,
|
|
1129
|
+
root_span_id: this.root_span_id,
|
|
1130
|
+
...this._object_info,
|
|
1131
|
+
[IS_MERGE_FIELD]: this.isMerge
|
|
1132
|
+
};
|
|
1133
|
+
this.internalData = {};
|
|
1134
|
+
this.bgLogger.log([record]);
|
|
1135
|
+
}
|
|
1136
|
+
startSpan(name, args) {
|
|
1137
|
+
this.checkNotFinished();
|
|
1138
|
+
return new _SpanImpl({
|
|
1139
|
+
bgLogger: this.bgLogger,
|
|
1140
|
+
name,
|
|
1141
|
+
...args,
|
|
1142
|
+
parentSpan: this
|
|
1143
|
+
});
|
|
1144
|
+
}
|
|
1145
|
+
traced(name, callback, args) {
|
|
1146
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
1147
|
+
const span = this.startSpan(name, argsRest);
|
|
1148
|
+
return runFinally(
|
|
1149
|
+
() => {
|
|
1150
|
+
if (setCurrent ?? true) {
|
|
1151
|
+
return withCurrent(span, () => callback(span));
|
|
1152
|
+
} else {
|
|
1153
|
+
return callback(span);
|
|
1154
|
+
}
|
|
1155
|
+
},
|
|
1156
|
+
() => span.end()
|
|
1157
|
+
);
|
|
1158
|
+
}
|
|
1159
|
+
end(args) {
|
|
1160
|
+
this.checkNotFinished();
|
|
1161
|
+
const endTime = args?.endTime ?? getCurrentUnixTimestamp();
|
|
1162
|
+
this.internalData = { metrics: { end: endTime } };
|
|
1163
|
+
this.log({});
|
|
1164
|
+
this.finished = true;
|
|
1165
|
+
unterminatedObjects.removeUnterminated(this);
|
|
1166
|
+
return endTime;
|
|
1167
|
+
}
|
|
1168
|
+
close(args) {
|
|
1169
|
+
return this.end(args);
|
|
1170
|
+
}
|
|
1171
|
+
checkNotFinished() {
|
|
1172
|
+
if (this.finished) {
|
|
1173
|
+
throw new Error("Cannot invoke method on finished span");
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
};
|
|
1177
|
+
async function _initDataset(project_name, {
|
|
1178
|
+
name,
|
|
1179
|
+
description,
|
|
1180
|
+
version
|
|
1181
|
+
} = {}) {
|
|
1182
|
+
const args = {
|
|
1183
|
+
org_id: _state.orgId,
|
|
1184
|
+
project_name,
|
|
1185
|
+
dataset_name: name,
|
|
1186
|
+
description
|
|
1187
|
+
};
|
|
1188
|
+
const response = await _state.apiConn().post_json("api/dataset/register", args);
|
|
1189
|
+
const project = response.project;
|
|
1190
|
+
const dataset = response.dataset;
|
|
1191
|
+
const user_id = (await _state.userInfo())["id"];
|
|
1192
|
+
return new Dataset(project, dataset.id, dataset.name, user_id, version);
|
|
1193
|
+
}
|
|
1194
|
+
var Dataset = class {
|
|
1195
|
+
constructor(project, id, name, user_id, pinnedVersion) {
|
|
1196
|
+
this._fetchedData = void 0;
|
|
1197
|
+
this.finished = false;
|
|
1198
|
+
this.project = project;
|
|
1199
|
+
this.id = id;
|
|
1200
|
+
this.name = name;
|
|
1201
|
+
this.user_id = user_id;
|
|
1202
|
+
this.pinnedVersion = pinnedVersion;
|
|
1203
|
+
this.logger = new BackgroundLogger();
|
|
1204
|
+
unterminatedObjects.addUnterminated(this, isomorph_default.getCallerLocation());
|
|
1205
|
+
}
|
|
1206
|
+
/**
|
|
1207
|
+
* Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
|
|
1208
|
+
* and a record with that `id` already exists, it will be overwritten (upsert).
|
|
1209
|
+
*
|
|
1210
|
+
* @param event The event to log.
|
|
1211
|
+
* @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
|
|
1212
|
+
* @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object).
|
|
1213
|
+
* @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
|
|
1214
|
+
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
1215
|
+
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
1216
|
+
* JSON-serializable type, but its keys must be strings.
|
|
1217
|
+
* @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
|
|
1218
|
+
* @returns The `id` of the logged record.
|
|
1219
|
+
*/
|
|
1220
|
+
insert({
|
|
1221
|
+
input,
|
|
1222
|
+
output,
|
|
1223
|
+
metadata,
|
|
1224
|
+
id
|
|
1225
|
+
}) {
|
|
1226
|
+
this.checkNotFinished();
|
|
1227
|
+
if (metadata !== void 0) {
|
|
1228
|
+
for (const key of Object.keys(metadata)) {
|
|
1229
|
+
if (typeof key !== "string") {
|
|
1230
|
+
throw new Error("metadata keys must be strings");
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
const args = {
|
|
1235
|
+
id: id || v4_default(),
|
|
1236
|
+
inputs: input,
|
|
1237
|
+
output,
|
|
1238
|
+
project_id: this.project.id,
|
|
1239
|
+
dataset_id: this.id,
|
|
1240
|
+
user_id: this.user_id,
|
|
1241
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1242
|
+
metadata
|
|
1243
|
+
};
|
|
1244
|
+
this.logger.log([args]);
|
|
1245
|
+
return args.id;
|
|
1246
|
+
}
|
|
1247
|
+
delete(id) {
|
|
1248
|
+
this.checkNotFinished();
|
|
1249
|
+
const user_id = this.user_id;
|
|
1250
|
+
const args = {
|
|
1251
|
+
id,
|
|
1252
|
+
project_id: this.project.id,
|
|
1253
|
+
dataset_id: this.id,
|
|
1254
|
+
user_id,
|
|
1255
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1256
|
+
_object_delete: true
|
|
1257
|
+
};
|
|
1258
|
+
this.logger.log([args]);
|
|
1259
|
+
return args.id;
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Summarize the dataset, including high level metrics about its size and other metadata.
|
|
1263
|
+
* @param summarizeData Whether to summarize the data. If false, only the metadata will be returned.
|
|
1264
|
+
* @returns `DatasetSummary`
|
|
1265
|
+
* @returns A summary of the dataset.
|
|
1266
|
+
*/
|
|
1267
|
+
async summarize(options = {}) {
|
|
1268
|
+
this.checkNotFinished();
|
|
1269
|
+
let { summarizeData = true } = options || {};
|
|
1270
|
+
await this.logger.flush();
|
|
1271
|
+
const projectUrl = `${_state.apiUrl}/app/${encodeURIComponent(
|
|
1272
|
+
_state.orgName
|
|
1273
|
+
)}/p/${encodeURIComponent(this.project.name)}`;
|
|
1274
|
+
const datasetUrl = `${projectUrl}/d/${encodeURIComponent(this.name)}`;
|
|
1275
|
+
let dataSummary = void 0;
|
|
1276
|
+
if (summarizeData) {
|
|
1277
|
+
dataSummary = await _state.logConn().get_json(
|
|
1278
|
+
"dataset-summary",
|
|
1279
|
+
{
|
|
1280
|
+
dataset_id: this.id
|
|
1281
|
+
},
|
|
1282
|
+
3
|
|
1283
|
+
);
|
|
1284
|
+
}
|
|
1285
|
+
return {
|
|
1286
|
+
projectName: this.project.name,
|
|
1287
|
+
datasetName: this.name,
|
|
1288
|
+
projectUrl,
|
|
1289
|
+
datasetUrl,
|
|
1290
|
+
dataSummary
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
/**
|
|
1294
|
+
* Fetch all records in the dataset.
|
|
1295
|
+
*
|
|
1296
|
+
* @example
|
|
1297
|
+
* ```
|
|
1298
|
+
* // Use an async iterator to fetch all records in the dataset.
|
|
1299
|
+
* for await (const record of dataset.fetch()) {
|
|
1300
|
+
* console.log(record);
|
|
1301
|
+
* }
|
|
1302
|
+
*
|
|
1303
|
+
* // You can also iterate over the dataset directly.
|
|
1304
|
+
* for await (const record of dataset) {
|
|
1305
|
+
* console.log(record);
|
|
1306
|
+
* }
|
|
1307
|
+
* ```
|
|
1308
|
+
*
|
|
1309
|
+
* @returns An iterator over the dataset's records.
|
|
1310
|
+
*/
|
|
1311
|
+
async *fetch() {
|
|
1312
|
+
this.checkNotFinished();
|
|
1313
|
+
const records = await this.fetchedData();
|
|
1314
|
+
for (const record of records) {
|
|
1315
|
+
yield {
|
|
1316
|
+
id: record.id,
|
|
1317
|
+
input: record.input && JSON.parse(record.input),
|
|
1318
|
+
output: record.input && JSON.parse(record.output),
|
|
1319
|
+
metadata: record.metadata && JSON.parse(record.metadata)
|
|
1320
|
+
};
|
|
1321
|
+
}
|
|
1322
|
+
this.clearCache();
|
|
1323
|
+
}
|
|
1324
|
+
/**
|
|
1325
|
+
* Fetch all records in the dataset.
|
|
1326
|
+
*
|
|
1327
|
+
* @example
|
|
1328
|
+
* ```
|
|
1329
|
+
* // Use an async iterator to fetch all records in the dataset.
|
|
1330
|
+
* for await (const record of dataset) {
|
|
1331
|
+
* console.log(record);
|
|
1332
|
+
* }
|
|
1333
|
+
* ```
|
|
1334
|
+
*/
|
|
1335
|
+
[Symbol.asyncIterator]() {
|
|
1336
|
+
this.checkNotFinished();
|
|
1337
|
+
return this.fetch();
|
|
1338
|
+
}
|
|
1339
|
+
async fetchedData() {
|
|
1340
|
+
this.checkNotFinished();
|
|
1341
|
+
if (this._fetchedData === void 0) {
|
|
1342
|
+
const resp = await _state.logConn().get("object/dataset", {
|
|
1343
|
+
id: this.id,
|
|
1344
|
+
fmt: "json",
|
|
1345
|
+
version: this.pinnedVersion
|
|
1346
|
+
});
|
|
1347
|
+
const text = await resp.text();
|
|
1348
|
+
this._fetchedData = text.split("\n").filter((x) => x.trim() !== "").map((x) => JSON.parse(x));
|
|
1349
|
+
}
|
|
1350
|
+
return this._fetchedData || [];
|
|
1351
|
+
}
|
|
1352
|
+
clearCache() {
|
|
1353
|
+
this.checkNotFinished();
|
|
1354
|
+
this._fetchedData = void 0;
|
|
1355
|
+
}
|
|
1356
|
+
async version() {
|
|
1357
|
+
this.checkNotFinished();
|
|
1358
|
+
if (this.pinnedVersion !== void 0) {
|
|
1359
|
+
return this.pinnedVersion;
|
|
1360
|
+
} else {
|
|
1361
|
+
const fetchedData = await this.fetchedData();
|
|
1362
|
+
let maxVersion = void 0;
|
|
1363
|
+
for (const record of fetchedData) {
|
|
1364
|
+
const xactId = record[TRANSACTION_ID_FIELD];
|
|
1365
|
+
if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
|
|
1366
|
+
maxVersion = xactId;
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
return maxVersion;
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
/**
|
|
1373
|
+
* Terminate connection to the dataset and return its id. After calling close, you may not invoke any further methods on the dataset object.
|
|
1374
|
+
*
|
|
1375
|
+
* Will be invoked automatically if the dataset is bound as a context manager.
|
|
1376
|
+
*
|
|
1377
|
+
* @returns The dataset id.
|
|
1378
|
+
*/
|
|
1379
|
+
async close() {
|
|
1380
|
+
this.checkNotFinished();
|
|
1381
|
+
await this.logger.flush();
|
|
1382
|
+
this.finished = true;
|
|
1383
|
+
unterminatedObjects.removeUnterminated(this);
|
|
1384
|
+
return this.id;
|
|
1385
|
+
}
|
|
1386
|
+
checkNotFinished() {
|
|
1387
|
+
if (this.finished) {
|
|
1388
|
+
throw new Error("Cannot invoke method on finished dataset");
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
};
|
|
1392
|
+
export {
|
|
1393
|
+
Dataset,
|
|
1394
|
+
Experiment,
|
|
1395
|
+
Logger,
|
|
1396
|
+
NoopSpan,
|
|
1397
|
+
SpanImpl,
|
|
1398
|
+
_internalGetGlobalState,
|
|
1399
|
+
currentExperiment,
|
|
1400
|
+
currentLogger,
|
|
1401
|
+
currentSpan,
|
|
1402
|
+
init,
|
|
1403
|
+
initDataset,
|
|
1404
|
+
initLogger,
|
|
1405
|
+
log,
|
|
1406
|
+
login,
|
|
1407
|
+
noopSpan,
|
|
1408
|
+
startSpan,
|
|
1409
|
+
summarize,
|
|
1410
|
+
traced,
|
|
1411
|
+
withCurrent,
|
|
1412
|
+
withDataset,
|
|
1413
|
+
withExperiment,
|
|
1414
|
+
withLogger
|
|
1415
|
+
};
|