braintrust 0.0.139 → 0.0.141-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +42 -0
- package/.turbo/turbo-watch.log +1843 -0
- package/LICENSE +201 -0
- package/braintrust-0.0.141.tgz +0 -0
- package/dist/{logger.d.ts → browser.d.mts} +194 -90
- package/dist/browser.d.ts +1089 -2
- package/dist/browser.js +444 -9231
- package/dist/browser.mjs +3203 -0
- package/dist/cli.js +3831 -27083
- package/dist/index.d.mts +1250 -0
- package/dist/index.d.ts +1236 -41
- package/dist/index.js +1769 -18971
- package/dist/index.mjs +5210 -0
- package/package.json +40 -23
- package/tsup.config.ts +23 -0
- package/turbo.json +8 -0
- package/vitest.config.js +6 -0
- package/dist/browser-config.d.ts +0 -6
- package/dist/cli.d.ts +0 -35
- package/dist/framework.d.ts +0 -180
- package/dist/framework.test.d.ts +0 -1
- package/dist/functions.d.ts +0 -15
- package/dist/gitutil.d.ts +0 -17
- package/dist/isomorph.d.ts +0 -27
- package/dist/jest/nodeModulesPaths.d.ts +0 -15
- package/dist/jest/tryRealpath.d.ts +0 -7
- package/dist/node.d.ts +0 -1
- package/dist/progress.d.ts +0 -18
- package/dist/stackutil.d.ts +0 -8
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/dist/util.d.ts +0 -11
- package/dist/wrappers/ai-sdk.d.ts +0 -8
- package/dist/wrappers/oai.d.ts +0 -34
- package/jest.config.js +0 -5
package/dist/browser.mjs
ADDED
|
@@ -0,0 +1,3203 @@
|
|
|
1
|
+
// src/isomorph.ts
|
|
2
|
+
var DefaultAsyncLocalStorage = class {
|
|
3
|
+
constructor() {
|
|
4
|
+
}
|
|
5
|
+
enterWith(_) {
|
|
6
|
+
}
|
|
7
|
+
run(_, callback) {
|
|
8
|
+
return callback();
|
|
9
|
+
}
|
|
10
|
+
getStore() {
|
|
11
|
+
return void 0;
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
var iso = {
|
|
15
|
+
getRepoInfo: async (_settings) => void 0,
|
|
16
|
+
getPastNAncestors: async () => [],
|
|
17
|
+
getEnv: (_name) => void 0,
|
|
18
|
+
getCallerLocation: () => void 0,
|
|
19
|
+
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
20
|
+
processOn: (_0, _1) => {
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
var isomorph_default = iso;
|
|
24
|
+
|
|
25
|
+
// src/logger.ts
|
|
26
|
+
import { v4 as uuidv4 } from "uuid";
|
|
27
|
+
import {
|
|
28
|
+
TRANSACTION_ID_FIELD,
|
|
29
|
+
IS_MERGE_FIELD,
|
|
30
|
+
mergeDicts,
|
|
31
|
+
mergeRowBatch,
|
|
32
|
+
VALID_SOURCES,
|
|
33
|
+
AUDIT_SOURCE_FIELD,
|
|
34
|
+
AUDIT_METADATA_FIELD,
|
|
35
|
+
mergeGitMetadataSettings,
|
|
36
|
+
DEFAULT_IS_LEGACY_DATASET,
|
|
37
|
+
ensureDatasetRecord,
|
|
38
|
+
makeLegacyEvent,
|
|
39
|
+
constructJsonArray,
|
|
40
|
+
SpanTypeAttribute,
|
|
41
|
+
batchItems,
|
|
42
|
+
SpanComponentsV2,
|
|
43
|
+
SpanObjectTypeV2,
|
|
44
|
+
SpanRowIdsV2,
|
|
45
|
+
gitMetadataSettingsSchema
|
|
46
|
+
} from "@braintrust/core";
|
|
47
|
+
import {
|
|
48
|
+
BRAINTRUST_PARAMS,
|
|
49
|
+
promptSchema,
|
|
50
|
+
toolsSchema
|
|
51
|
+
} from "@braintrust/core/typespecs";
|
|
52
|
+
|
|
53
|
+
// src/util.ts
|
|
54
|
+
var GLOBAL_PROJECT = "Global";
|
|
55
|
+
function runFinally(f, finallyF) {
|
|
56
|
+
let runSyncCleanup = true;
|
|
57
|
+
try {
|
|
58
|
+
const ret = f();
|
|
59
|
+
if (ret instanceof Promise) {
|
|
60
|
+
runSyncCleanup = false;
|
|
61
|
+
return ret.finally(finallyF);
|
|
62
|
+
} else {
|
|
63
|
+
return ret;
|
|
64
|
+
}
|
|
65
|
+
} finally {
|
|
66
|
+
if (runSyncCleanup) {
|
|
67
|
+
finallyF();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function getCurrentUnixTimestamp() {
|
|
72
|
+
return (/* @__PURE__ */ new Date()).getTime() / 1e3;
|
|
73
|
+
}
|
|
74
|
+
function isEmpty(a) {
|
|
75
|
+
return a === void 0 || a === null;
|
|
76
|
+
}
|
|
77
|
+
var LazyValue = class {
|
|
78
|
+
callable;
|
|
79
|
+
value = {
|
|
80
|
+
hasComputed: false
|
|
81
|
+
};
|
|
82
|
+
constructor(callable) {
|
|
83
|
+
this.callable = callable;
|
|
84
|
+
}
|
|
85
|
+
get() {
|
|
86
|
+
if (this.value.hasComputed) {
|
|
87
|
+
return this.value.val;
|
|
88
|
+
}
|
|
89
|
+
this.value = { hasComputed: true, val: this.callable() };
|
|
90
|
+
return this.value.val;
|
|
91
|
+
}
|
|
92
|
+
get hasComputed() {
|
|
93
|
+
return this.value.hasComputed;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
// src/logger.ts
|
|
98
|
+
import Mustache from "mustache";
|
|
99
|
+
import { z } from "zod";
|
|
100
|
+
|
|
101
|
+
// src/stream.ts
|
|
102
|
+
import { callEventSchema } from "@braintrust/core/typespecs";
|
|
103
|
+
import {
|
|
104
|
+
createParser
|
|
105
|
+
} from "eventsource-parser";
|
|
106
|
+
var BraintrustStream = class _BraintrustStream {
|
|
107
|
+
stream;
|
|
108
|
+
constructor(baseStream) {
|
|
109
|
+
this.stream = baseStream.pipeThrough(btStreamParser());
|
|
110
|
+
}
|
|
111
|
+
copy() {
|
|
112
|
+
const [newStream, copyStream] = this.stream.tee();
|
|
113
|
+
this.stream = copyStream;
|
|
114
|
+
return new _BraintrustStream(newStream);
|
|
115
|
+
}
|
|
116
|
+
toReadableStream() {
|
|
117
|
+
return this.stream;
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
function btStreamParser() {
|
|
121
|
+
const decoder = new TextDecoder();
|
|
122
|
+
let parser;
|
|
123
|
+
return new TransformStream({
|
|
124
|
+
async start(controller) {
|
|
125
|
+
parser = createParser((event) => {
|
|
126
|
+
if (event.type === "reconnect-interval") {
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const parsed = callEventSchema.safeParse(event);
|
|
130
|
+
if (!parsed.success) {
|
|
131
|
+
throw new Error(`Failed to parse event: ${parsed.error}`);
|
|
132
|
+
}
|
|
133
|
+
switch (event.event) {
|
|
134
|
+
case "text_delta":
|
|
135
|
+
controller.enqueue({
|
|
136
|
+
type: "text_delta",
|
|
137
|
+
data: JSON.parse(event.data)
|
|
138
|
+
});
|
|
139
|
+
break;
|
|
140
|
+
case "json_delta":
|
|
141
|
+
controller.enqueue({
|
|
142
|
+
type: "json_delta",
|
|
143
|
+
data: event.data
|
|
144
|
+
});
|
|
145
|
+
break;
|
|
146
|
+
case "done":
|
|
147
|
+
break;
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
},
|
|
151
|
+
async transform(chunk, controller) {
|
|
152
|
+
if (chunk instanceof Uint8Array) {
|
|
153
|
+
parser.feed(decoder.decode(chunk));
|
|
154
|
+
} else if (typeof chunk === "string") {
|
|
155
|
+
parser.feed(chunk);
|
|
156
|
+
} else {
|
|
157
|
+
controller.enqueue(chunk);
|
|
158
|
+
}
|
|
159
|
+
},
|
|
160
|
+
async flush(controller) {
|
|
161
|
+
controller.terminate();
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
function createFinalValuePassThroughStream(onFinal) {
|
|
166
|
+
const decoder = new TextDecoder();
|
|
167
|
+
const textChunks = [];
|
|
168
|
+
const jsonChunks = [];
|
|
169
|
+
const transformStream = new TransformStream({
|
|
170
|
+
transform(chunk, controller) {
|
|
171
|
+
if (typeof chunk === "string") {
|
|
172
|
+
textChunks.push(chunk);
|
|
173
|
+
} else if (chunk instanceof Uint8Array) {
|
|
174
|
+
textChunks.push(decoder.decode(chunk));
|
|
175
|
+
} else {
|
|
176
|
+
const chunkType = chunk.type;
|
|
177
|
+
switch (chunkType) {
|
|
178
|
+
case "text_delta":
|
|
179
|
+
textChunks.push(chunk.data);
|
|
180
|
+
break;
|
|
181
|
+
case "json_delta":
|
|
182
|
+
jsonChunks.push(chunk.data);
|
|
183
|
+
break;
|
|
184
|
+
default:
|
|
185
|
+
const _type = chunkType;
|
|
186
|
+
throw new Error(`Unknown chunk type ${_type}`);
|
|
187
|
+
}
|
|
188
|
+
controller.enqueue(chunk);
|
|
189
|
+
}
|
|
190
|
+
},
|
|
191
|
+
flush(controller) {
|
|
192
|
+
if (jsonChunks.length > 0) {
|
|
193
|
+
onFinal(JSON.parse(jsonChunks.join("")));
|
|
194
|
+
} else if (textChunks.length > 0) {
|
|
195
|
+
onFinal(textChunks.join(""));
|
|
196
|
+
} else {
|
|
197
|
+
onFinal(void 0);
|
|
198
|
+
}
|
|
199
|
+
controller.terminate();
|
|
200
|
+
}
|
|
201
|
+
});
|
|
202
|
+
return transformStream;
|
|
203
|
+
}
|
|
204
|
+
function devNullWritableStream() {
|
|
205
|
+
return new WritableStream({
|
|
206
|
+
write(chunk) {
|
|
207
|
+
},
|
|
208
|
+
close() {
|
|
209
|
+
},
|
|
210
|
+
abort(reason) {
|
|
211
|
+
},
|
|
212
|
+
start(controller) {
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// src/logger.ts
|
|
218
|
+
import { waitUntil } from "@vercel/functions";
|
|
219
|
+
var NoopSpan = class {
|
|
220
|
+
id;
|
|
221
|
+
kind = "span";
|
|
222
|
+
constructor() {
|
|
223
|
+
this.id = "";
|
|
224
|
+
}
|
|
225
|
+
log(_) {
|
|
226
|
+
}
|
|
227
|
+
logFeedback(_event) {
|
|
228
|
+
}
|
|
229
|
+
traced(callback, _1) {
|
|
230
|
+
return callback(this);
|
|
231
|
+
}
|
|
232
|
+
startSpan(_1) {
|
|
233
|
+
return this;
|
|
234
|
+
}
|
|
235
|
+
end(args) {
|
|
236
|
+
return args?.endTime ?? getCurrentUnixTimestamp();
|
|
237
|
+
}
|
|
238
|
+
async export() {
|
|
239
|
+
return "";
|
|
240
|
+
}
|
|
241
|
+
async flush() {
|
|
242
|
+
}
|
|
243
|
+
close(args) {
|
|
244
|
+
return this.end(args);
|
|
245
|
+
}
|
|
246
|
+
setAttributes(_args) {
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
var NOOP_SPAN = new NoopSpan();
|
|
250
|
+
var loginSchema = z.strictObject({
|
|
251
|
+
appUrl: z.string(),
|
|
252
|
+
appPublicUrl: z.string(),
|
|
253
|
+
orgName: z.string(),
|
|
254
|
+
logUrl: z.string(),
|
|
255
|
+
proxyUrl: z.string(),
|
|
256
|
+
loginToken: z.string(),
|
|
257
|
+
orgId: z.string().nullish(),
|
|
258
|
+
gitMetadataSettings: gitMetadataSettingsSchema.nullish()
|
|
259
|
+
});
|
|
260
|
+
var BraintrustState = class _BraintrustState {
|
|
261
|
+
constructor(loginParams) {
|
|
262
|
+
this.loginParams = loginParams;
|
|
263
|
+
this.id = (/* @__PURE__ */ new Date()).toLocaleString();
|
|
264
|
+
this.currentExperiment = void 0;
|
|
265
|
+
this.currentLogger = void 0;
|
|
266
|
+
this.currentSpan = isomorph_default.newAsyncLocalStorage();
|
|
267
|
+
if (loginParams.fetch) {
|
|
268
|
+
this.fetch = loginParams.fetch;
|
|
269
|
+
}
|
|
270
|
+
const defaultGetLogConn = async () => {
|
|
271
|
+
await this.login({});
|
|
272
|
+
return this.logConn();
|
|
273
|
+
};
|
|
274
|
+
this._bgLogger = new BackgroundLogger(new LazyValue(defaultGetLogConn));
|
|
275
|
+
this.resetLoginInfo();
|
|
276
|
+
}
|
|
277
|
+
id;
|
|
278
|
+
currentExperiment;
|
|
279
|
+
// Note: the value of IsAsyncFlush doesn't really matter here, since we
|
|
280
|
+
// (safely) dynamically cast it whenever retrieving the logger.
|
|
281
|
+
currentLogger;
|
|
282
|
+
currentSpan;
|
|
283
|
+
// Any time we re-log in, we directly update the logConn inside the logger.
|
|
284
|
+
// This is preferable to replacing the whole logger, which would create the
|
|
285
|
+
// possibility of multiple loggers floating around, which may not log in a
|
|
286
|
+
// deterministic order.
|
|
287
|
+
_bgLogger;
|
|
288
|
+
appUrl = null;
|
|
289
|
+
appPublicUrl = null;
|
|
290
|
+
loginToken = null;
|
|
291
|
+
orgId = null;
|
|
292
|
+
orgName = null;
|
|
293
|
+
logUrl = null;
|
|
294
|
+
proxyUrl = null;
|
|
295
|
+
loggedIn = false;
|
|
296
|
+
gitMetadataSettings;
|
|
297
|
+
fetch = globalThis.fetch;
|
|
298
|
+
_apiConn = null;
|
|
299
|
+
_logConn = null;
|
|
300
|
+
_proxyConn = null;
|
|
301
|
+
resetLoginInfo() {
|
|
302
|
+
this.appUrl = null;
|
|
303
|
+
this.appPublicUrl = null;
|
|
304
|
+
this.loginToken = null;
|
|
305
|
+
this.orgId = null;
|
|
306
|
+
this.orgName = null;
|
|
307
|
+
this.logUrl = null;
|
|
308
|
+
this.proxyUrl = null;
|
|
309
|
+
this.loggedIn = false;
|
|
310
|
+
this.gitMetadataSettings = void 0;
|
|
311
|
+
this._apiConn = null;
|
|
312
|
+
this._logConn = null;
|
|
313
|
+
this._proxyConn = null;
|
|
314
|
+
}
|
|
315
|
+
copyLoginInfo(other) {
|
|
316
|
+
this.appUrl = other.appUrl;
|
|
317
|
+
this.appPublicUrl = other.appPublicUrl;
|
|
318
|
+
this.loginToken = other.loginToken;
|
|
319
|
+
this.orgId = other.orgId;
|
|
320
|
+
this.orgName = other.orgName;
|
|
321
|
+
this.logUrl = other.logUrl;
|
|
322
|
+
this.proxyUrl = other.proxyUrl;
|
|
323
|
+
this.loggedIn = other.loggedIn;
|
|
324
|
+
this.gitMetadataSettings = other.gitMetadataSettings;
|
|
325
|
+
this._apiConn = other._apiConn;
|
|
326
|
+
this._logConn = other._logConn;
|
|
327
|
+
this._proxyConn = other._proxyConn;
|
|
328
|
+
}
|
|
329
|
+
serialize() {
|
|
330
|
+
if (!this.loggedIn) {
|
|
331
|
+
throw new Error(
|
|
332
|
+
"Cannot serialize BraintrustState without being logged in"
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
if (!this.appUrl || !this.appPublicUrl || !this.logUrl || !this.proxyUrl || !this.orgName || !this.loginToken || !this.loggedIn) {
|
|
336
|
+
throw new Error(
|
|
337
|
+
"Cannot serialize BraintrustState without all login attributes"
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
return {
|
|
341
|
+
appUrl: this.appUrl,
|
|
342
|
+
appPublicUrl: this.appPublicUrl,
|
|
343
|
+
loginToken: this.loginToken,
|
|
344
|
+
orgId: this.orgId,
|
|
345
|
+
orgName: this.orgName,
|
|
346
|
+
logUrl: this.logUrl,
|
|
347
|
+
proxyUrl: this.proxyUrl,
|
|
348
|
+
gitMetadataSettings: this.gitMetadataSettings
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
static deserialize(serialized) {
|
|
352
|
+
const serializedParsed = loginSchema.safeParse(serialized);
|
|
353
|
+
if (!serializedParsed.success) {
|
|
354
|
+
throw new Error(
|
|
355
|
+
`Cannot deserialize BraintrustState: ${serializedParsed.error.errors}`
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
const state = new _BraintrustState({});
|
|
359
|
+
for (const key of Object.keys(loginSchema.shape)) {
|
|
360
|
+
state[key] = serializedParsed.data[key];
|
|
361
|
+
}
|
|
362
|
+
if (!state.loginToken) {
|
|
363
|
+
throw new Error(
|
|
364
|
+
"Cannot deserialize BraintrustState without a login token"
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
state.logConn().set_token(state.loginToken);
|
|
368
|
+
state.logConn().make_long_lived();
|
|
369
|
+
state.apiConn().set_token(state.loginToken);
|
|
370
|
+
state.proxyConn().set_token(state.loginToken);
|
|
371
|
+
state.loggedIn = true;
|
|
372
|
+
state.loginReplaceLogConn(state.logConn());
|
|
373
|
+
return state;
|
|
374
|
+
}
|
|
375
|
+
setFetch(fetch) {
|
|
376
|
+
this.loginParams.fetch = fetch;
|
|
377
|
+
this.fetch = fetch;
|
|
378
|
+
this._logConn?.setFetch(fetch);
|
|
379
|
+
this._apiConn?.setFetch(fetch);
|
|
380
|
+
}
|
|
381
|
+
async login(loginParams) {
|
|
382
|
+
if (this.logUrl && !loginParams.forceLogin) {
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
const newState = await loginToState({
|
|
386
|
+
...this.loginParams,
|
|
387
|
+
...loginParams
|
|
388
|
+
});
|
|
389
|
+
this.copyLoginInfo(newState);
|
|
390
|
+
}
|
|
391
|
+
apiConn() {
|
|
392
|
+
if (!this._apiConn) {
|
|
393
|
+
if (!this.appUrl) {
|
|
394
|
+
throw new Error("Must initialize appUrl before requesting apiConn");
|
|
395
|
+
}
|
|
396
|
+
this._apiConn = new HTTPConnection(this.appUrl, this.fetch);
|
|
397
|
+
}
|
|
398
|
+
return this._apiConn;
|
|
399
|
+
}
|
|
400
|
+
logConn() {
|
|
401
|
+
if (!this._logConn) {
|
|
402
|
+
if (!this.logUrl) {
|
|
403
|
+
throw new Error("Must initialize logUrl before requesting logConn");
|
|
404
|
+
}
|
|
405
|
+
this._logConn = new HTTPConnection(this.logUrl, this.fetch);
|
|
406
|
+
}
|
|
407
|
+
return this._logConn;
|
|
408
|
+
}
|
|
409
|
+
proxyConn() {
|
|
410
|
+
if (!this._proxyConn) {
|
|
411
|
+
if (!this.proxyUrl) {
|
|
412
|
+
throw new Error("Must initialize proxyUrl before requesting proxyConn");
|
|
413
|
+
}
|
|
414
|
+
this._proxyConn = new HTTPConnection(this.proxyUrl, this.fetch);
|
|
415
|
+
}
|
|
416
|
+
return this._proxyConn;
|
|
417
|
+
}
|
|
418
|
+
bgLogger() {
|
|
419
|
+
return this._bgLogger;
|
|
420
|
+
}
|
|
421
|
+
// Should only be called by the login function.
|
|
422
|
+
loginReplaceLogConn(logConn) {
|
|
423
|
+
this._bgLogger.internalReplaceLogConn(logConn);
|
|
424
|
+
}
|
|
425
|
+
};
|
|
426
|
+
var _globalState;
|
|
427
|
+
function _internalSetInitialState() {
|
|
428
|
+
if (_globalState) {
|
|
429
|
+
throw new Error("Cannot set initial state more than once");
|
|
430
|
+
}
|
|
431
|
+
_globalState = globalThis.__inherited_braintrust_state || new BraintrustState({
|
|
432
|
+
/*empty login options*/
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
var _internalGetGlobalState = () => _globalState;
|
|
436
|
+
var FailedHTTPResponse = class extends Error {
|
|
437
|
+
status;
|
|
438
|
+
text;
|
|
439
|
+
data;
|
|
440
|
+
constructor(status, text, data = null) {
|
|
441
|
+
super(`${status}: ${text}`);
|
|
442
|
+
this.status = status;
|
|
443
|
+
this.text = text;
|
|
444
|
+
this.data = data;
|
|
445
|
+
}
|
|
446
|
+
};
|
|
447
|
+
async function checkResponse(resp) {
|
|
448
|
+
if (resp.ok) {
|
|
449
|
+
return resp;
|
|
450
|
+
} else {
|
|
451
|
+
throw new FailedHTTPResponse(
|
|
452
|
+
resp.status,
|
|
453
|
+
resp.statusText,
|
|
454
|
+
await resp.text()
|
|
455
|
+
);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
var HTTPConnection = class _HTTPConnection {
|
|
459
|
+
base_url;
|
|
460
|
+
token;
|
|
461
|
+
headers;
|
|
462
|
+
fetch;
|
|
463
|
+
constructor(base_url, fetch) {
|
|
464
|
+
this.base_url = base_url;
|
|
465
|
+
this.token = null;
|
|
466
|
+
this.headers = {};
|
|
467
|
+
this._reset();
|
|
468
|
+
this.fetch = fetch;
|
|
469
|
+
}
|
|
470
|
+
setFetch(fetch) {
|
|
471
|
+
this.fetch = fetch;
|
|
472
|
+
}
|
|
473
|
+
async ping() {
|
|
474
|
+
try {
|
|
475
|
+
const resp = await this.get("ping");
|
|
476
|
+
return resp.status === 200;
|
|
477
|
+
} catch (e) {
|
|
478
|
+
return false;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
make_long_lived() {
|
|
482
|
+
this._reset();
|
|
483
|
+
}
|
|
484
|
+
static sanitize_token(token) {
|
|
485
|
+
return token.trim();
|
|
486
|
+
}
|
|
487
|
+
set_token(token) {
|
|
488
|
+
token = _HTTPConnection.sanitize_token(token);
|
|
489
|
+
this.token = token;
|
|
490
|
+
this._reset();
|
|
491
|
+
}
|
|
492
|
+
// As far as I can tell, you cannot set the retry/backoff factor here
|
|
493
|
+
_reset() {
|
|
494
|
+
this.headers = {};
|
|
495
|
+
if (this.token) {
|
|
496
|
+
this.headers["Authorization"] = `Bearer ${this.token}`;
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
async get(path, params = void 0, config) {
|
|
500
|
+
const { headers, ...rest } = config || {};
|
|
501
|
+
const url = new URL(_urljoin(this.base_url, path));
|
|
502
|
+
url.search = new URLSearchParams(
|
|
503
|
+
params ? Object.fromEntries(
|
|
504
|
+
Object.entries(params).filter(([_, v]) => v !== void 0)
|
|
505
|
+
) : {}
|
|
506
|
+
).toString();
|
|
507
|
+
return await checkResponse(
|
|
508
|
+
// Using toString() here makes it work with isomorphic fetch
|
|
509
|
+
await this.fetch(url.toString(), {
|
|
510
|
+
headers: {
|
|
511
|
+
Accept: "application/json",
|
|
512
|
+
...this.headers,
|
|
513
|
+
...headers
|
|
514
|
+
},
|
|
515
|
+
keepalive: true,
|
|
516
|
+
...rest
|
|
517
|
+
})
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
async post(path, params, config) {
|
|
521
|
+
const { headers, ...rest } = config || {};
|
|
522
|
+
return await checkResponse(
|
|
523
|
+
await this.fetch(_urljoin(this.base_url, path), {
|
|
524
|
+
method: "POST",
|
|
525
|
+
headers: {
|
|
526
|
+
Accept: "application/json",
|
|
527
|
+
"Content-Type": "application/json",
|
|
528
|
+
...this.headers,
|
|
529
|
+
...headers
|
|
530
|
+
},
|
|
531
|
+
body: typeof params === "string" ? params : params ? JSON.stringify(params) : void 0,
|
|
532
|
+
keepalive: true,
|
|
533
|
+
...rest
|
|
534
|
+
})
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
async get_json(object_type, args = void 0, retries = 0) {
|
|
538
|
+
const tries = retries + 1;
|
|
539
|
+
for (let i = 0; i < tries; i++) {
|
|
540
|
+
try {
|
|
541
|
+
const resp = await this.get(`${object_type}`, args);
|
|
542
|
+
return await resp.json();
|
|
543
|
+
} catch (e) {
|
|
544
|
+
if (i < tries - 1) {
|
|
545
|
+
console.log(
|
|
546
|
+
`Retrying API request ${object_type} ${JSON.stringify(args)} ${e.status} ${e.text}`
|
|
547
|
+
);
|
|
548
|
+
continue;
|
|
549
|
+
}
|
|
550
|
+
throw e;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
async post_json(object_type, args = void 0) {
|
|
555
|
+
const resp = await this.post(`${object_type}`, args, {
|
|
556
|
+
headers: { "Content-Type": "application/json" }
|
|
557
|
+
});
|
|
558
|
+
return await resp.json();
|
|
559
|
+
}
|
|
560
|
+
};
|
|
561
|
+
function logFeedbackImpl(state, parentObjectType, parentObjectId, {
|
|
562
|
+
id,
|
|
563
|
+
expected,
|
|
564
|
+
scores,
|
|
565
|
+
metadata: inputMetadata,
|
|
566
|
+
tags,
|
|
567
|
+
comment,
|
|
568
|
+
source: inputSource
|
|
569
|
+
}) {
|
|
570
|
+
const source = inputSource ?? "external";
|
|
571
|
+
if (!VALID_SOURCES.includes(source)) {
|
|
572
|
+
throw new Error(`source must be one of ${VALID_SOURCES}`);
|
|
573
|
+
}
|
|
574
|
+
if (isEmpty(scores) && isEmpty(expected) && isEmpty(tags) && isEmpty(comment)) {
|
|
575
|
+
throw new Error(
|
|
576
|
+
"At least one of scores, expected, tags, or comment must be specified"
|
|
577
|
+
);
|
|
578
|
+
}
|
|
579
|
+
const validatedEvent = validateAndSanitizeExperimentLogPartialArgs({
|
|
580
|
+
scores,
|
|
581
|
+
metadata: inputMetadata,
|
|
582
|
+
expected,
|
|
583
|
+
tags
|
|
584
|
+
});
|
|
585
|
+
let { metadata, ...updateEvent } = validatedEvent;
|
|
586
|
+
updateEvent = Object.fromEntries(
|
|
587
|
+
Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
|
|
588
|
+
);
|
|
589
|
+
const parentIds = async () => new SpanComponentsV2({
|
|
590
|
+
objectType: parentObjectType,
|
|
591
|
+
objectId: await parentObjectId.get()
|
|
592
|
+
}).objectIdFields();
|
|
593
|
+
if (Object.keys(updateEvent).length > 0) {
|
|
594
|
+
const record = new LazyValue(async () => {
|
|
595
|
+
return {
|
|
596
|
+
id,
|
|
597
|
+
...updateEvent,
|
|
598
|
+
...await parentIds(),
|
|
599
|
+
[AUDIT_SOURCE_FIELD]: source,
|
|
600
|
+
[AUDIT_METADATA_FIELD]: metadata,
|
|
601
|
+
[IS_MERGE_FIELD]: true
|
|
602
|
+
};
|
|
603
|
+
});
|
|
604
|
+
state.bgLogger().log([record]);
|
|
605
|
+
}
|
|
606
|
+
if (!isEmpty(comment)) {
|
|
607
|
+
const record = new LazyValue(async () => {
|
|
608
|
+
return {
|
|
609
|
+
id: uuidv4(),
|
|
610
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
611
|
+
origin: {
|
|
612
|
+
// NOTE: We do not know (or care?) what the transaction id of the row that
|
|
613
|
+
// we're commenting on is here, so we omit it.
|
|
614
|
+
id
|
|
615
|
+
},
|
|
616
|
+
comment: {
|
|
617
|
+
text: comment
|
|
618
|
+
},
|
|
619
|
+
...await parentIds(),
|
|
620
|
+
[AUDIT_SOURCE_FIELD]: source,
|
|
621
|
+
[AUDIT_METADATA_FIELD]: metadata
|
|
622
|
+
};
|
|
623
|
+
});
|
|
624
|
+
state.bgLogger().log([record]);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
function spanComponentsToObjectIdLambda(state, components) {
|
|
628
|
+
if (components.objectId) {
|
|
629
|
+
const ret = components.objectId;
|
|
630
|
+
return async () => ret;
|
|
631
|
+
}
|
|
632
|
+
if (!components.computeObjectMetadataArgs) {
|
|
633
|
+
throw new Error(
|
|
634
|
+
"Impossible: must provide either objectId or computeObjectMetadataArgs"
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
switch (components.objectType) {
|
|
638
|
+
case SpanObjectTypeV2.EXPERIMENT:
|
|
639
|
+
throw new Error(
|
|
640
|
+
"Impossible: computeObjectMetadataArgs not supported for experiments"
|
|
641
|
+
);
|
|
642
|
+
case SpanObjectTypeV2.PROJECT_LOGS:
|
|
643
|
+
return async () => (await computeLoggerMetadata(state, {
|
|
644
|
+
...components.computeObjectMetadataArgs
|
|
645
|
+
})).project.id;
|
|
646
|
+
default:
|
|
647
|
+
const x = components.objectType;
|
|
648
|
+
throw new Error(`Unknown object type: ${x}`);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
function startSpanParentArgs(args) {
|
|
652
|
+
let argParentObjectId = void 0;
|
|
653
|
+
let argParentSpanIds = void 0;
|
|
654
|
+
if (args.parent) {
|
|
655
|
+
if (args.parentSpanIds) {
|
|
656
|
+
throw new Error("Cannot specify both parent and parentSpanIds");
|
|
657
|
+
}
|
|
658
|
+
const parentComponents = SpanComponentsV2.fromStr(args.parent);
|
|
659
|
+
if (args.parentObjectType !== parentComponents.objectType) {
|
|
660
|
+
throw new Error(
|
|
661
|
+
`Mismatch between expected span parent object type ${args.parentObjectType} and provided type ${parentComponents.objectType}`
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
const parentComponentsObjectIdLambda = spanComponentsToObjectIdLambda(
|
|
665
|
+
args.state,
|
|
666
|
+
parentComponents
|
|
667
|
+
);
|
|
668
|
+
const computeParentObjectId = async () => {
|
|
669
|
+
const parentComponentsObjectId = await parentComponentsObjectIdLambda();
|
|
670
|
+
if (await args.parentObjectId.get() !== parentComponentsObjectId) {
|
|
671
|
+
throw new Error(
|
|
672
|
+
`Mismatch between expected span parent object id ${await args.parentObjectId.get()} and provided id ${parentComponentsObjectId}`
|
|
673
|
+
);
|
|
674
|
+
}
|
|
675
|
+
return await args.parentObjectId.get();
|
|
676
|
+
};
|
|
677
|
+
argParentObjectId = new LazyValue(computeParentObjectId);
|
|
678
|
+
if (parentComponents.rowIds) {
|
|
679
|
+
argParentSpanIds = {
|
|
680
|
+
spanId: parentComponents.rowIds.spanId,
|
|
681
|
+
rootSpanId: parentComponents.rowIds.rootSpanId
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
} else {
|
|
685
|
+
argParentObjectId = args.parentObjectId;
|
|
686
|
+
argParentSpanIds = args.parentSpanIds;
|
|
687
|
+
}
|
|
688
|
+
return {
|
|
689
|
+
parentObjectType: args.parentObjectType,
|
|
690
|
+
parentObjectId: argParentObjectId,
|
|
691
|
+
parentComputeObjectMetadataArgs: args.parentComputeObjectMetadataArgs,
|
|
692
|
+
parentSpanIds: argParentSpanIds
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
var Logger = class {
|
|
696
|
+
state;
|
|
697
|
+
lazyMetadata;
|
|
698
|
+
_asyncFlush;
|
|
699
|
+
computeMetadataArgs;
|
|
700
|
+
lastStartTime;
|
|
701
|
+
lazyId;
|
|
702
|
+
calledStartSpan;
|
|
703
|
+
// For type identification.
|
|
704
|
+
kind = "logger";
|
|
705
|
+
constructor(state, lazyMetadata, logOptions = {}) {
|
|
706
|
+
this.lazyMetadata = lazyMetadata;
|
|
707
|
+
this._asyncFlush = logOptions.asyncFlush;
|
|
708
|
+
this.computeMetadataArgs = logOptions.computeMetadataArgs;
|
|
709
|
+
this.lastStartTime = getCurrentUnixTimestamp();
|
|
710
|
+
this.lazyId = new LazyValue(async () => await this.id);
|
|
711
|
+
this.calledStartSpan = false;
|
|
712
|
+
this.state = state;
|
|
713
|
+
}
|
|
714
|
+
get org_id() {
|
|
715
|
+
return (async () => {
|
|
716
|
+
return (await this.lazyMetadata.get()).org_id;
|
|
717
|
+
})();
|
|
718
|
+
}
|
|
719
|
+
get project() {
|
|
720
|
+
return (async () => {
|
|
721
|
+
return (await this.lazyMetadata.get()).project;
|
|
722
|
+
})();
|
|
723
|
+
}
|
|
724
|
+
get id() {
|
|
725
|
+
return (async () => (await this.project).id)();
|
|
726
|
+
}
|
|
727
|
+
parentObjectType() {
|
|
728
|
+
return SpanObjectTypeV2.PROJECT_LOGS;
|
|
729
|
+
}
|
|
730
|
+
triggerWaitUntilFlush() {
|
|
731
|
+
if (!this.state.bgLogger().syncFlush) {
|
|
732
|
+
return waitUntil(this.state.bgLogger().flush());
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
/**
|
|
736
|
+
* Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
|
|
737
|
+
*
|
|
738
|
+
* @param event The event to log.
|
|
739
|
+
* @param event.input: (Optional) the arguments that uniquely define a user input (an arbitrary, JSON serializable object).
|
|
740
|
+
* @param event.output: (Optional) the output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
|
|
741
|
+
* @param event.expected: (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
742
|
+
* @param event.scores: (Optional) a dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs.
|
|
743
|
+
* @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
|
|
744
|
+
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
|
|
745
|
+
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
746
|
+
* @param options Additional logging options
|
|
747
|
+
* @param options.allowConcurrentWithSpans in rare cases where you need to log at the top level separately from spans on the logger elsewhere, set this to true.
|
|
748
|
+
* :returns: The `id` of the logged event.
|
|
749
|
+
*/
|
|
750
|
+
log(event, options) {
|
|
751
|
+
if (this.calledStartSpan && !options?.allowConcurrentWithSpans) {
|
|
752
|
+
throw new Error(
|
|
753
|
+
"Cannot run toplevel `log` method while using spans. To log to the span, call `logger.traced` and then log with `span.log`"
|
|
754
|
+
);
|
|
755
|
+
}
|
|
756
|
+
const span = this.startSpanImpl({ startTime: this.lastStartTime, event });
|
|
757
|
+
this.lastStartTime = span.end();
|
|
758
|
+
const ret = span.id;
|
|
759
|
+
if (this.asyncFlush === true) {
|
|
760
|
+
this.triggerWaitUntilFlush();
|
|
761
|
+
return ret;
|
|
762
|
+
} else {
|
|
763
|
+
return (async () => {
|
|
764
|
+
await this.flush();
|
|
765
|
+
return ret;
|
|
766
|
+
})();
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
/**
|
|
770
|
+
* Create a new toplevel span underneath the logger. The name defaults to "root".
|
|
771
|
+
*
|
|
772
|
+
* See `Span.traced` for full details.
|
|
773
|
+
*/
|
|
774
|
+
traced(callback, args) {
|
|
775
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
776
|
+
const span = this.startSpan(argsRest);
|
|
777
|
+
const ret = runFinally(
|
|
778
|
+
() => {
|
|
779
|
+
if (setCurrent ?? true) {
|
|
780
|
+
return withCurrent(span, callback);
|
|
781
|
+
} else {
|
|
782
|
+
return callback(span);
|
|
783
|
+
}
|
|
784
|
+
},
|
|
785
|
+
() => span.end()
|
|
786
|
+
);
|
|
787
|
+
if (this.asyncFlush) {
|
|
788
|
+
this.triggerWaitUntilFlush();
|
|
789
|
+
return ret;
|
|
790
|
+
} else {
|
|
791
|
+
return (async () => {
|
|
792
|
+
const awaitedRet = await ret;
|
|
793
|
+
await this.flush();
|
|
794
|
+
return awaitedRet;
|
|
795
|
+
})();
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
/**
|
|
799
|
+
* Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
|
|
800
|
+
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
801
|
+
* so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
|
|
802
|
+
*
|
|
803
|
+
* See `traced` for full details.
|
|
804
|
+
*/
|
|
805
|
+
startSpan(args) {
|
|
806
|
+
this.calledStartSpan = true;
|
|
807
|
+
return this.startSpanImpl(args);
|
|
808
|
+
}
|
|
809
|
+
startSpanImpl(args) {
|
|
810
|
+
return new SpanImpl({
|
|
811
|
+
state: this.state,
|
|
812
|
+
...startSpanParentArgs({
|
|
813
|
+
state: this.state,
|
|
814
|
+
parent: args?.parent,
|
|
815
|
+
parentObjectType: this.parentObjectType(),
|
|
816
|
+
parentObjectId: this.lazyId,
|
|
817
|
+
parentComputeObjectMetadataArgs: this.computeMetadataArgs,
|
|
818
|
+
parentSpanIds: void 0
|
|
819
|
+
}),
|
|
820
|
+
...args,
|
|
821
|
+
defaultRootType: SpanTypeAttribute.TASK
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* Log feedback to an event. Feedback is used to save feedback scores, set an expected value, or add a comment.
|
|
826
|
+
*
|
|
827
|
+
* @param event
|
|
828
|
+
* @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
|
|
829
|
+
* @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
|
|
830
|
+
* @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
|
|
831
|
+
* @param event.comment (Optional) an optional comment string to log about the event.
|
|
832
|
+
* @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
|
|
833
|
+
* @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
|
|
834
|
+
*/
|
|
835
|
+
logFeedback(event) {
|
|
836
|
+
logFeedbackImpl(this.state, this.parentObjectType(), this.lazyId, event);
|
|
837
|
+
}
|
|
838
|
+
/**
|
|
839
|
+
* Return a serialized representation of the logger that can be used to start subspans in other places. See `Span.start_span` for more details.
|
|
840
|
+
*/
|
|
841
|
+
async export() {
|
|
842
|
+
let objectId = void 0;
|
|
843
|
+
let computeObjectMetadataArgs = void 0;
|
|
844
|
+
if (this.computeMetadataArgs && !this.lazyId.hasComputed) {
|
|
845
|
+
computeObjectMetadataArgs = this.computeMetadataArgs;
|
|
846
|
+
} else {
|
|
847
|
+
objectId = await this.lazyId.get();
|
|
848
|
+
}
|
|
849
|
+
return new SpanComponentsV2({
|
|
850
|
+
objectType: this.parentObjectType(),
|
|
851
|
+
objectId,
|
|
852
|
+
computeObjectMetadataArgs
|
|
853
|
+
}).toStr();
|
|
854
|
+
}
|
|
855
|
+
/*
|
|
856
|
+
* Flush any pending logs to the server.
|
|
857
|
+
*/
|
|
858
|
+
async flush() {
|
|
859
|
+
return await this.state.bgLogger().flush();
|
|
860
|
+
}
|
|
861
|
+
get asyncFlush() {
|
|
862
|
+
return this._asyncFlush;
|
|
863
|
+
}
|
|
864
|
+
};
|
|
865
|
+
function castLogger(logger, asyncFlush) {
|
|
866
|
+
if (logger === void 0)
|
|
867
|
+
return void 0;
|
|
868
|
+
if (asyncFlush && !!asyncFlush !== !!logger.asyncFlush) {
|
|
869
|
+
throw new Error(
|
|
870
|
+
`Asserted asyncFlush setting ${asyncFlush} does not match stored logger's setting ${logger.asyncFlush}`
|
|
871
|
+
);
|
|
872
|
+
}
|
|
873
|
+
return logger;
|
|
874
|
+
}
|
|
875
|
+
function constructLogs3Data(items) {
|
|
876
|
+
return `{"rows": ${constructJsonArray(items)}, "api_version": 2}`;
|
|
877
|
+
}
|
|
878
|
+
function now() {
|
|
879
|
+
return (/* @__PURE__ */ new Date()).getTime();
|
|
880
|
+
}
|
|
881
|
+
var BackgroundLogger = class _BackgroundLogger {
|
|
882
|
+
logConn;
|
|
883
|
+
items = [];
|
|
884
|
+
activeFlush = Promise.resolve();
|
|
885
|
+
activeFlushResolved = true;
|
|
886
|
+
activeFlushError = void 0;
|
|
887
|
+
syncFlush = false;
|
|
888
|
+
// 6 MB for the AWS lambda gateway (from our own testing).
|
|
889
|
+
maxRequestSize = 6 * 1024 * 1024;
|
|
890
|
+
defaultBatchSize = 100;
|
|
891
|
+
numTries = 3;
|
|
892
|
+
queueDropExceedingMaxsize = void 0;
|
|
893
|
+
queueDropLoggingPeriod = 60;
|
|
894
|
+
failedPublishPayloadsDir = void 0;
|
|
895
|
+
allPublishPayloadsDir = void 0;
|
|
896
|
+
queueDropLoggingState = {
|
|
897
|
+
numDropped: 0,
|
|
898
|
+
lastLoggedTimestamp: 0
|
|
899
|
+
};
|
|
900
|
+
constructor(logConn) {
|
|
901
|
+
this.logConn = logConn;
|
|
902
|
+
const syncFlushEnv = Number(isomorph_default.getEnv("BRAINTRUST_SYNC_FLUSH"));
|
|
903
|
+
if (!isNaN(syncFlushEnv)) {
|
|
904
|
+
this.syncFlush = Boolean(syncFlushEnv);
|
|
905
|
+
}
|
|
906
|
+
const defaultBatchSizeEnv = Number(
|
|
907
|
+
isomorph_default.getEnv("BRAINTRUST_DEFAULT_BATCH_SIZE")
|
|
908
|
+
);
|
|
909
|
+
if (!isNaN(defaultBatchSizeEnv)) {
|
|
910
|
+
this.defaultBatchSize = defaultBatchSizeEnv;
|
|
911
|
+
}
|
|
912
|
+
const maxRequestSizeEnv = Number(isomorph_default.getEnv("BRAINTRUST_MAX_REQUEST_SIZE"));
|
|
913
|
+
if (!isNaN(maxRequestSizeEnv)) {
|
|
914
|
+
this.maxRequestSize = maxRequestSizeEnv;
|
|
915
|
+
}
|
|
916
|
+
const numTriesEnv = Number(isomorph_default.getEnv("BRAINTRUST_NUM_RETRIES"));
|
|
917
|
+
if (!isNaN(numTriesEnv)) {
|
|
918
|
+
this.numTries = numTriesEnv + 1;
|
|
919
|
+
}
|
|
920
|
+
const queueDropExceedingMaxsizeEnv = Number(
|
|
921
|
+
isomorph_default.getEnv("BRAINTRUST_QUEUE_DROP_EXCEEDING_MAXSIZE")
|
|
922
|
+
);
|
|
923
|
+
if (!isNaN(queueDropExceedingMaxsizeEnv)) {
|
|
924
|
+
this.queueDropExceedingMaxsize = queueDropExceedingMaxsizeEnv;
|
|
925
|
+
}
|
|
926
|
+
const queueDropLoggingPeriodEnv = Number(
|
|
927
|
+
isomorph_default.getEnv("BRAINTRUST_QUEUE_DROP_LOGGING_PERIOD")
|
|
928
|
+
);
|
|
929
|
+
if (!isNaN(queueDropLoggingPeriodEnv)) {
|
|
930
|
+
this.queueDropLoggingPeriod = queueDropLoggingPeriodEnv;
|
|
931
|
+
}
|
|
932
|
+
const failedPublishPayloadsDirEnv = isomorph_default.getEnv(
|
|
933
|
+
"BRAINTRUST_FAILED_PUBLISH_PAYLOADS_DIR"
|
|
934
|
+
);
|
|
935
|
+
if (failedPublishPayloadsDirEnv) {
|
|
936
|
+
this.failedPublishPayloadsDir = failedPublishPayloadsDirEnv;
|
|
937
|
+
}
|
|
938
|
+
const allPublishPayloadsDirEnv = isomorph_default.getEnv(
|
|
939
|
+
"BRAINTRUST_ALL_PUBLISH_PAYLOADS_DIR"
|
|
940
|
+
);
|
|
941
|
+
if (allPublishPayloadsDirEnv) {
|
|
942
|
+
this.allPublishPayloadsDir = allPublishPayloadsDirEnv;
|
|
943
|
+
}
|
|
944
|
+
isomorph_default.processOn("beforeExit", async () => {
|
|
945
|
+
await this.flush();
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
log(items) {
|
|
949
|
+
const [addedItems, droppedItems] = (() => {
|
|
950
|
+
if (this.queueDropExceedingMaxsize === void 0) {
|
|
951
|
+
return [items, []];
|
|
952
|
+
}
|
|
953
|
+
const numElementsToAdd = Math.min(
|
|
954
|
+
Math.max(this.queueDropExceedingMaxsize - this.items.length, 0),
|
|
955
|
+
items.length
|
|
956
|
+
);
|
|
957
|
+
return [items.slice(0, numElementsToAdd), items.slice(numElementsToAdd)];
|
|
958
|
+
})();
|
|
959
|
+
this.items.push(...addedItems);
|
|
960
|
+
if (!this.syncFlush) {
|
|
961
|
+
this.triggerActiveFlush();
|
|
962
|
+
}
|
|
963
|
+
if (droppedItems.length) {
|
|
964
|
+
this.registerDroppedItemCount(droppedItems.length);
|
|
965
|
+
if (this.allPublishPayloadsDir || this.failedPublishPayloadsDir) {
|
|
966
|
+
this.dumpDroppedEvents(droppedItems);
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
async flush() {
|
|
971
|
+
if (this.syncFlush) {
|
|
972
|
+
this.triggerActiveFlush();
|
|
973
|
+
}
|
|
974
|
+
await this.activeFlush;
|
|
975
|
+
if (this.activeFlushError) {
|
|
976
|
+
const err = this.activeFlushError;
|
|
977
|
+
this.activeFlushError = void 0;
|
|
978
|
+
throw err;
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
async flushOnce(args) {
|
|
982
|
+
const batchSize = args?.batchSize ?? this.defaultBatchSize;
|
|
983
|
+
const wrappedItems = this.items;
|
|
984
|
+
this.items = [];
|
|
985
|
+
const allItems = await this.unwrapLazyValues(wrappedItems);
|
|
986
|
+
if (allItems.length === 0) {
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
const allItemsStr = allItems.map(
|
|
990
|
+
(bucket) => bucket.map((item) => JSON.stringify(item))
|
|
991
|
+
);
|
|
992
|
+
const batchSets = batchItems({
|
|
993
|
+
items: allItemsStr,
|
|
994
|
+
batchMaxNumItems: batchSize,
|
|
995
|
+
batchMaxNumBytes: this.maxRequestSize / 2
|
|
996
|
+
});
|
|
997
|
+
for (const batchSet of batchSets) {
|
|
998
|
+
const postPromises = batchSet.map(
|
|
999
|
+
(batch) => (async () => {
|
|
1000
|
+
try {
|
|
1001
|
+
await this.submitLogsRequest(batch);
|
|
1002
|
+
return { type: "success" };
|
|
1003
|
+
} catch (e) {
|
|
1004
|
+
return { type: "error", value: e };
|
|
1005
|
+
}
|
|
1006
|
+
})()
|
|
1007
|
+
);
|
|
1008
|
+
const results = await Promise.all(postPromises);
|
|
1009
|
+
const failingResultErrors = results.map((r) => r.type === "success" ? void 0 : r.value).filter((r) => r !== void 0);
|
|
1010
|
+
if (failingResultErrors.length) {
|
|
1011
|
+
throw new AggregateError(
|
|
1012
|
+
failingResultErrors,
|
|
1013
|
+
`Encountered the following errors while logging:`
|
|
1014
|
+
);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
if (this.items.length > 0) {
|
|
1018
|
+
await this.flushOnce(args);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
async unwrapLazyValues(wrappedItems) {
|
|
1022
|
+
for (let i = 0; i < this.numTries; ++i) {
|
|
1023
|
+
try {
|
|
1024
|
+
const itemPromises = wrappedItems.map((x) => x.get());
|
|
1025
|
+
return mergeRowBatch(await Promise.all(itemPromises));
|
|
1026
|
+
} catch (e) {
|
|
1027
|
+
let errmsg = "Encountered error when constructing records to flush";
|
|
1028
|
+
const isRetrying = i + 1 < this.numTries;
|
|
1029
|
+
if (isRetrying) {
|
|
1030
|
+
errmsg += ". Retrying";
|
|
1031
|
+
}
|
|
1032
|
+
console.warn(errmsg);
|
|
1033
|
+
if (!isRetrying && this.syncFlush) {
|
|
1034
|
+
throw e;
|
|
1035
|
+
} else {
|
|
1036
|
+
console.warn(e);
|
|
1037
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
console.warn(
|
|
1042
|
+
`Failed to construct log records to flush after ${this.numTries} attempts. Dropping batch`
|
|
1043
|
+
);
|
|
1044
|
+
return [];
|
|
1045
|
+
}
|
|
1046
|
+
async submitLogsRequest(items) {
|
|
1047
|
+
const conn = await this.logConn.get();
|
|
1048
|
+
const dataStr = constructLogs3Data(items);
|
|
1049
|
+
if (this.allPublishPayloadsDir) {
|
|
1050
|
+
await _BackgroundLogger.writePayloadToDir({
|
|
1051
|
+
payloadDir: this.allPublishPayloadsDir,
|
|
1052
|
+
payload: dataStr
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
for (let i = 0; i < this.numTries; i++) {
|
|
1056
|
+
const startTime = now();
|
|
1057
|
+
let error = void 0;
|
|
1058
|
+
try {
|
|
1059
|
+
await conn.post_json("logs3", dataStr);
|
|
1060
|
+
} catch (e) {
|
|
1061
|
+
try {
|
|
1062
|
+
const legacyDataS = constructJsonArray(
|
|
1063
|
+
items.map(
|
|
1064
|
+
(r) => JSON.stringify(makeLegacyEvent(JSON.parse(r)))
|
|
1065
|
+
)
|
|
1066
|
+
);
|
|
1067
|
+
await conn.post_json("logs", legacyDataS);
|
|
1068
|
+
} catch (e2) {
|
|
1069
|
+
error = e2;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
if (error === void 0) {
|
|
1073
|
+
return;
|
|
1074
|
+
}
|
|
1075
|
+
const isRetrying = i + 1 < this.numTries;
|
|
1076
|
+
const retryingText = isRetrying ? "" : " Retrying";
|
|
1077
|
+
const errorText = (() => {
|
|
1078
|
+
if (error instanceof FailedHTTPResponse) {
|
|
1079
|
+
return `${error.status} (${error.text}): ${error.data}`;
|
|
1080
|
+
} else {
|
|
1081
|
+
return `${error}`;
|
|
1082
|
+
}
|
|
1083
|
+
})();
|
|
1084
|
+
const errMsg = `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}.${retryingText}
|
|
1085
|
+
Error: ${errorText}`;
|
|
1086
|
+
if (!isRetrying && this.failedPublishPayloadsDir) {
|
|
1087
|
+
await _BackgroundLogger.writePayloadToDir({
|
|
1088
|
+
payloadDir: this.failedPublishPayloadsDir,
|
|
1089
|
+
payload: dataStr
|
|
1090
|
+
});
|
|
1091
|
+
this.logFailedPayloadsDir();
|
|
1092
|
+
}
|
|
1093
|
+
if (!isRetrying && this.syncFlush) {
|
|
1094
|
+
throw new Error(errMsg);
|
|
1095
|
+
} else {
|
|
1096
|
+
console.warn(errMsg);
|
|
1097
|
+
if (isRetrying) {
|
|
1098
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
console.warn(
|
|
1103
|
+
`log request failed after ${this.numTries} retries. Dropping batch`
|
|
1104
|
+
);
|
|
1105
|
+
return;
|
|
1106
|
+
}
|
|
1107
|
+
registerDroppedItemCount(numItems) {
|
|
1108
|
+
if (numItems <= 0) {
|
|
1109
|
+
return;
|
|
1110
|
+
}
|
|
1111
|
+
this.queueDropLoggingState.numDropped += numItems;
|
|
1112
|
+
const timeNow = getCurrentUnixTimestamp();
|
|
1113
|
+
if (timeNow - this.queueDropLoggingState.lastLoggedTimestamp > this.queueDropLoggingPeriod) {
|
|
1114
|
+
console.warn(
|
|
1115
|
+
`Dropped ${this.queueDropLoggingState.numDropped} elements due to full queue`
|
|
1116
|
+
);
|
|
1117
|
+
if (this.failedPublishPayloadsDir) {
|
|
1118
|
+
this.logFailedPayloadsDir();
|
|
1119
|
+
}
|
|
1120
|
+
this.queueDropLoggingState.numDropped = 0;
|
|
1121
|
+
this.queueDropLoggingState.lastLoggedTimestamp = timeNow;
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
async dumpDroppedEvents(wrappedItems) {
|
|
1125
|
+
const publishPayloadsDir = [
|
|
1126
|
+
this.allPublishPayloadsDir,
|
|
1127
|
+
this.failedPublishPayloadsDir
|
|
1128
|
+
].reduce((acc, x) => x ? acc.concat([x]) : acc, new Array());
|
|
1129
|
+
if (!(wrappedItems.length && publishPayloadsDir.length)) {
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
try {
|
|
1133
|
+
const allItems = await this.unwrapLazyValues(wrappedItems);
|
|
1134
|
+
const dataStr = constructLogs3Data(
|
|
1135
|
+
allItems.map((x) => JSON.stringify(x))
|
|
1136
|
+
);
|
|
1137
|
+
for (const payloadDir of publishPayloadsDir) {
|
|
1138
|
+
await _BackgroundLogger.writePayloadToDir({
|
|
1139
|
+
payloadDir,
|
|
1140
|
+
payload: dataStr
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
} catch (e) {
|
|
1144
|
+
console.error(e);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
static async writePayloadToDir({
|
|
1148
|
+
payloadDir,
|
|
1149
|
+
payload
|
|
1150
|
+
}) {
|
|
1151
|
+
if (!(isomorph_default.pathJoin && isomorph_default.mkdir && isomorph_default.writeFile)) {
|
|
1152
|
+
console.warn(
|
|
1153
|
+
"Cannot dump payloads: filesystem-operations not supported on this platform"
|
|
1154
|
+
);
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1157
|
+
const payloadFile = isomorph_default.pathJoin(
|
|
1158
|
+
payloadDir,
|
|
1159
|
+
`payload_${getCurrentUnixTimestamp()}_${uuidv4().slice(0, 8)}.json`
|
|
1160
|
+
);
|
|
1161
|
+
try {
|
|
1162
|
+
await isomorph_default.mkdir(payloadDir, { recursive: true });
|
|
1163
|
+
await isomorph_default.writeFile(payloadFile, payload);
|
|
1164
|
+
} catch (e) {
|
|
1165
|
+
console.error(
|
|
1166
|
+
`Failed to write failed payload to output file ${payloadFile}:
|
|
1167
|
+
`,
|
|
1168
|
+
e
|
|
1169
|
+
);
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
triggerActiveFlush() {
|
|
1173
|
+
if (this.activeFlushResolved) {
|
|
1174
|
+
this.activeFlushResolved = false;
|
|
1175
|
+
this.activeFlushError = void 0;
|
|
1176
|
+
this.activeFlush = (async () => {
|
|
1177
|
+
try {
|
|
1178
|
+
await this.flushOnce();
|
|
1179
|
+
} catch (err) {
|
|
1180
|
+
this.activeFlushError = err;
|
|
1181
|
+
} finally {
|
|
1182
|
+
this.activeFlushResolved = true;
|
|
1183
|
+
}
|
|
1184
|
+
})();
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
logFailedPayloadsDir() {
|
|
1188
|
+
console.warn(`Logging failed payloads to ${this.failedPublishPayloadsDir}`);
|
|
1189
|
+
}
|
|
1190
|
+
// Should only be called by BraintrustState.
|
|
1191
|
+
internalReplaceLogConn(logConn) {
|
|
1192
|
+
this.logConn = new LazyValue(async () => logConn);
|
|
1193
|
+
}
|
|
1194
|
+
};
|
|
1195
|
+
function init(projectOrOptions, optionalOptions) {
|
|
1196
|
+
const options = (() => {
|
|
1197
|
+
if (typeof projectOrOptions === "string") {
|
|
1198
|
+
return { ...optionalOptions, project: projectOrOptions };
|
|
1199
|
+
} else {
|
|
1200
|
+
if (optionalOptions !== void 0) {
|
|
1201
|
+
throw new Error(
|
|
1202
|
+
"Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
|
|
1203
|
+
);
|
|
1204
|
+
}
|
|
1205
|
+
return projectOrOptions;
|
|
1206
|
+
}
|
|
1207
|
+
})();
|
|
1208
|
+
const {
|
|
1209
|
+
project,
|
|
1210
|
+
experiment,
|
|
1211
|
+
description,
|
|
1212
|
+
dataset,
|
|
1213
|
+
baseExperiment,
|
|
1214
|
+
isPublic,
|
|
1215
|
+
open,
|
|
1216
|
+
update,
|
|
1217
|
+
appUrl,
|
|
1218
|
+
apiKey,
|
|
1219
|
+
orgName,
|
|
1220
|
+
forceLogin,
|
|
1221
|
+
fetch,
|
|
1222
|
+
metadata,
|
|
1223
|
+
gitMetadataSettings,
|
|
1224
|
+
projectId,
|
|
1225
|
+
baseExperimentId,
|
|
1226
|
+
repoInfo,
|
|
1227
|
+
state: stateArg
|
|
1228
|
+
} = options;
|
|
1229
|
+
if (open && update) {
|
|
1230
|
+
throw new Error("Cannot open and update an experiment at the same time");
|
|
1231
|
+
}
|
|
1232
|
+
const state = stateArg ?? _globalState;
|
|
1233
|
+
if (open) {
|
|
1234
|
+
if (isEmpty(experiment)) {
|
|
1235
|
+
throw new Error(`Cannot open an experiment without specifying its name`);
|
|
1236
|
+
}
|
|
1237
|
+
const lazyMetadata2 = new LazyValue(
|
|
1238
|
+
async () => {
|
|
1239
|
+
await state.login({ apiKey, appUrl, orgName, fetch, forceLogin });
|
|
1240
|
+
const args = {
|
|
1241
|
+
project_name: project,
|
|
1242
|
+
project_id: projectId,
|
|
1243
|
+
org_name: state.orgName,
|
|
1244
|
+
experiment_name: experiment
|
|
1245
|
+
};
|
|
1246
|
+
const response = await state.apiConn().post_json("api/experiment/get", args);
|
|
1247
|
+
if (response.length === 0) {
|
|
1248
|
+
throw new Error(
|
|
1249
|
+
`Experiment ${experiment} not found in project ${projectId ?? project}.`
|
|
1250
|
+
);
|
|
1251
|
+
}
|
|
1252
|
+
const info = response[0];
|
|
1253
|
+
return {
|
|
1254
|
+
project: {
|
|
1255
|
+
id: info.project_id,
|
|
1256
|
+
name: project ?? "UNKNOWN_PROJECT",
|
|
1257
|
+
fullInfo: {}
|
|
1258
|
+
},
|
|
1259
|
+
experiment: {
|
|
1260
|
+
id: info.id,
|
|
1261
|
+
name: info.name,
|
|
1262
|
+
fullInfo: info
|
|
1263
|
+
}
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1266
|
+
);
|
|
1267
|
+
return new ReadonlyExperiment(
|
|
1268
|
+
stateArg ?? _globalState,
|
|
1269
|
+
lazyMetadata2
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
const lazyMetadata = new LazyValue(
|
|
1273
|
+
async () => {
|
|
1274
|
+
await state.login({ apiKey, appUrl, orgName });
|
|
1275
|
+
const args = {
|
|
1276
|
+
project_name: project,
|
|
1277
|
+
project_id: projectId,
|
|
1278
|
+
org_id: state.orgId,
|
|
1279
|
+
update
|
|
1280
|
+
};
|
|
1281
|
+
if (experiment) {
|
|
1282
|
+
args["experiment_name"] = experiment;
|
|
1283
|
+
}
|
|
1284
|
+
if (description) {
|
|
1285
|
+
args["description"] = description;
|
|
1286
|
+
}
|
|
1287
|
+
const repoInfoArg = await (async () => {
|
|
1288
|
+
if (repoInfo) {
|
|
1289
|
+
return repoInfo;
|
|
1290
|
+
}
|
|
1291
|
+
let mergedGitMetadataSettings = {
|
|
1292
|
+
...state.gitMetadataSettings || {
|
|
1293
|
+
collect: "all"
|
|
1294
|
+
}
|
|
1295
|
+
};
|
|
1296
|
+
if (gitMetadataSettings) {
|
|
1297
|
+
mergedGitMetadataSettings = mergeGitMetadataSettings(
|
|
1298
|
+
mergedGitMetadataSettings,
|
|
1299
|
+
gitMetadataSettings
|
|
1300
|
+
);
|
|
1301
|
+
}
|
|
1302
|
+
return await isomorph_default.getRepoInfo(mergedGitMetadataSettings);
|
|
1303
|
+
})();
|
|
1304
|
+
if (repoInfoArg) {
|
|
1305
|
+
args["repo_info"] = repoInfoArg;
|
|
1306
|
+
}
|
|
1307
|
+
if (baseExperimentId) {
|
|
1308
|
+
args["base_exp_id"] = baseExperimentId;
|
|
1309
|
+
} else if (baseExperiment) {
|
|
1310
|
+
args["base_experiment"] = baseExperiment;
|
|
1311
|
+
} else {
|
|
1312
|
+
args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
|
|
1313
|
+
}
|
|
1314
|
+
if (dataset !== void 0) {
|
|
1315
|
+
args["dataset_id"] = await dataset.id;
|
|
1316
|
+
args["dataset_version"] = await dataset.version();
|
|
1317
|
+
}
|
|
1318
|
+
if (isPublic !== void 0) {
|
|
1319
|
+
args["public"] = isPublic;
|
|
1320
|
+
}
|
|
1321
|
+
if (metadata) {
|
|
1322
|
+
args["metadata"] = metadata;
|
|
1323
|
+
}
|
|
1324
|
+
let response = null;
|
|
1325
|
+
while (true) {
|
|
1326
|
+
try {
|
|
1327
|
+
response = await state.apiConn().post_json("api/experiment/register", args);
|
|
1328
|
+
break;
|
|
1329
|
+
} catch (e) {
|
|
1330
|
+
if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
|
|
1331
|
+
console.warn(
|
|
1332
|
+
`Base experiment ${args["base_experiment"]} not found.`
|
|
1333
|
+
);
|
|
1334
|
+
delete args["base_experiment"];
|
|
1335
|
+
} else {
|
|
1336
|
+
throw e;
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
return {
|
|
1341
|
+
project: {
|
|
1342
|
+
id: response.project.id,
|
|
1343
|
+
name: response.project.name,
|
|
1344
|
+
fullInfo: response.project
|
|
1345
|
+
},
|
|
1346
|
+
experiment: {
|
|
1347
|
+
id: response.experiment.id,
|
|
1348
|
+
name: response.experiment.name,
|
|
1349
|
+
fullInfo: response.experiment
|
|
1350
|
+
}
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1353
|
+
);
|
|
1354
|
+
const ret = new Experiment(state, lazyMetadata, dataset);
|
|
1355
|
+
if (options.setCurrent ?? true) {
|
|
1356
|
+
state.currentExperiment = ret;
|
|
1357
|
+
}
|
|
1358
|
+
return ret;
|
|
1359
|
+
}
|
|
1360
|
+
function initExperiment(projectOrOptions, optionalOptions) {
|
|
1361
|
+
const options = (() => {
|
|
1362
|
+
if (typeof projectOrOptions === "string") {
|
|
1363
|
+
return { ...optionalOptions, project: projectOrOptions };
|
|
1364
|
+
} else {
|
|
1365
|
+
if (optionalOptions !== void 0) {
|
|
1366
|
+
throw new Error(
|
|
1367
|
+
"Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
return projectOrOptions;
|
|
1371
|
+
}
|
|
1372
|
+
})();
|
|
1373
|
+
return init(options);
|
|
1374
|
+
}
|
|
1375
|
+
function withExperiment(project, callback, options = {}) {
|
|
1376
|
+
console.warn(
|
|
1377
|
+
"withExperiment is deprecated and will be removed in a future version of braintrust. Simply create the experiment with `init`."
|
|
1378
|
+
);
|
|
1379
|
+
const experiment = init(project, options);
|
|
1380
|
+
return callback(experiment);
|
|
1381
|
+
}
|
|
1382
|
+
function withLogger(callback, options = {}) {
|
|
1383
|
+
console.warn(
|
|
1384
|
+
"withLogger is deprecated and will be removed in a future version of braintrust. Simply create the logger with `initLogger`."
|
|
1385
|
+
);
|
|
1386
|
+
const logger = initLogger(options);
|
|
1387
|
+
return callback(logger);
|
|
1388
|
+
}
|
|
1389
|
+
function initDataset(projectOrOptions, optionalOptions) {
|
|
1390
|
+
const options = (() => {
|
|
1391
|
+
if (typeof projectOrOptions === "string") {
|
|
1392
|
+
return { ...optionalOptions, project: projectOrOptions };
|
|
1393
|
+
} else {
|
|
1394
|
+
if (optionalOptions !== void 0) {
|
|
1395
|
+
throw new Error(
|
|
1396
|
+
"Cannot specify options struct as both parameters. Must call either initDataset(project, options) or initDataset(options)."
|
|
1397
|
+
);
|
|
1398
|
+
}
|
|
1399
|
+
return projectOrOptions;
|
|
1400
|
+
}
|
|
1401
|
+
})();
|
|
1402
|
+
const {
|
|
1403
|
+
project,
|
|
1404
|
+
dataset,
|
|
1405
|
+
description,
|
|
1406
|
+
version,
|
|
1407
|
+
appUrl,
|
|
1408
|
+
apiKey,
|
|
1409
|
+
orgName,
|
|
1410
|
+
fetch,
|
|
1411
|
+
forceLogin,
|
|
1412
|
+
projectId,
|
|
1413
|
+
useOutput: legacy,
|
|
1414
|
+
state: stateArg
|
|
1415
|
+
} = options;
|
|
1416
|
+
const state = stateArg ?? _globalState;
|
|
1417
|
+
const lazyMetadata = new LazyValue(
|
|
1418
|
+
async () => {
|
|
1419
|
+
await state.login({
|
|
1420
|
+
orgName,
|
|
1421
|
+
apiKey,
|
|
1422
|
+
appUrl,
|
|
1423
|
+
fetch,
|
|
1424
|
+
forceLogin
|
|
1425
|
+
});
|
|
1426
|
+
const args = {
|
|
1427
|
+
org_id: state.orgId,
|
|
1428
|
+
project_name: project,
|
|
1429
|
+
project_id: projectId,
|
|
1430
|
+
dataset_name: dataset,
|
|
1431
|
+
description
|
|
1432
|
+
};
|
|
1433
|
+
const response = await state.apiConn().post_json("api/dataset/register", args);
|
|
1434
|
+
return {
|
|
1435
|
+
project: {
|
|
1436
|
+
id: response.project.id,
|
|
1437
|
+
name: response.project.name,
|
|
1438
|
+
fullInfo: response.project
|
|
1439
|
+
},
|
|
1440
|
+
dataset: {
|
|
1441
|
+
id: response.dataset.id,
|
|
1442
|
+
name: response.dataset.name,
|
|
1443
|
+
fullInfo: response.dataset
|
|
1444
|
+
}
|
|
1445
|
+
};
|
|
1446
|
+
}
|
|
1447
|
+
);
|
|
1448
|
+
return new Dataset(stateArg ?? _globalState, lazyMetadata, version, legacy);
|
|
1449
|
+
}
|
|
1450
|
+
function withDataset(project, callback, options = {}) {
|
|
1451
|
+
console.warn(
|
|
1452
|
+
"withDataset is deprecated and will be removed in a future version of braintrust. Simply create the dataset with `initDataset`."
|
|
1453
|
+
);
|
|
1454
|
+
const dataset = initDataset(project, options);
|
|
1455
|
+
return callback(dataset);
|
|
1456
|
+
}
|
|
1457
|
+
async function computeLoggerMetadata(state, {
|
|
1458
|
+
project_name,
|
|
1459
|
+
project_id
|
|
1460
|
+
}) {
|
|
1461
|
+
const org_id = state.orgId;
|
|
1462
|
+
if (isEmpty(project_id)) {
|
|
1463
|
+
const response = await state.apiConn().post_json("api/project/register", {
|
|
1464
|
+
project_name: project_name || GLOBAL_PROJECT,
|
|
1465
|
+
org_id
|
|
1466
|
+
});
|
|
1467
|
+
return {
|
|
1468
|
+
org_id,
|
|
1469
|
+
project: {
|
|
1470
|
+
id: response.project.id,
|
|
1471
|
+
name: response.project.name,
|
|
1472
|
+
fullInfo: response.project
|
|
1473
|
+
}
|
|
1474
|
+
};
|
|
1475
|
+
} else if (isEmpty(project_name)) {
|
|
1476
|
+
const response = await state.apiConn().get_json("api/project", {
|
|
1477
|
+
id: project_id
|
|
1478
|
+
});
|
|
1479
|
+
return {
|
|
1480
|
+
org_id,
|
|
1481
|
+
project: {
|
|
1482
|
+
id: project_id,
|
|
1483
|
+
name: response.name,
|
|
1484
|
+
fullInfo: response.project
|
|
1485
|
+
}
|
|
1486
|
+
};
|
|
1487
|
+
} else {
|
|
1488
|
+
return {
|
|
1489
|
+
org_id,
|
|
1490
|
+
project: { id: project_id, name: project_name, fullInfo: {} }
|
|
1491
|
+
};
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
function initLogger(options = {}) {
|
|
1495
|
+
const {
|
|
1496
|
+
projectName,
|
|
1497
|
+
projectId,
|
|
1498
|
+
asyncFlush,
|
|
1499
|
+
appUrl,
|
|
1500
|
+
apiKey,
|
|
1501
|
+
orgName,
|
|
1502
|
+
forceLogin,
|
|
1503
|
+
fetch,
|
|
1504
|
+
state: stateArg
|
|
1505
|
+
} = options || {};
|
|
1506
|
+
const computeMetadataArgs = {
|
|
1507
|
+
project_name: projectName,
|
|
1508
|
+
project_id: projectId
|
|
1509
|
+
};
|
|
1510
|
+
const state = stateArg ?? _globalState;
|
|
1511
|
+
const lazyMetadata = new LazyValue(
|
|
1512
|
+
async () => {
|
|
1513
|
+
await state.login({
|
|
1514
|
+
orgName,
|
|
1515
|
+
apiKey,
|
|
1516
|
+
appUrl,
|
|
1517
|
+
forceLogin,
|
|
1518
|
+
fetch
|
|
1519
|
+
});
|
|
1520
|
+
return computeLoggerMetadata(state, computeMetadataArgs);
|
|
1521
|
+
}
|
|
1522
|
+
);
|
|
1523
|
+
const ret = new Logger(state, lazyMetadata, {
|
|
1524
|
+
asyncFlush,
|
|
1525
|
+
computeMetadataArgs
|
|
1526
|
+
});
|
|
1527
|
+
if (options.setCurrent ?? true) {
|
|
1528
|
+
state.currentLogger = ret;
|
|
1529
|
+
}
|
|
1530
|
+
return ret;
|
|
1531
|
+
}
|
|
1532
|
+
async function loadPrompt({
|
|
1533
|
+
projectName,
|
|
1534
|
+
projectId,
|
|
1535
|
+
slug,
|
|
1536
|
+
version,
|
|
1537
|
+
defaults,
|
|
1538
|
+
noTrace = false,
|
|
1539
|
+
appUrl,
|
|
1540
|
+
apiKey,
|
|
1541
|
+
orgName,
|
|
1542
|
+
fetch,
|
|
1543
|
+
forceLogin,
|
|
1544
|
+
state: stateArg
|
|
1545
|
+
}) {
|
|
1546
|
+
if (isEmpty(projectName) && isEmpty(projectId)) {
|
|
1547
|
+
throw new Error("Must specify either projectName or projectId");
|
|
1548
|
+
}
|
|
1549
|
+
if (isEmpty(slug)) {
|
|
1550
|
+
throw new Error("Must specify slug");
|
|
1551
|
+
}
|
|
1552
|
+
const state = stateArg ?? _globalState;
|
|
1553
|
+
state.login({
|
|
1554
|
+
orgName,
|
|
1555
|
+
apiKey,
|
|
1556
|
+
appUrl,
|
|
1557
|
+
fetch,
|
|
1558
|
+
forceLogin
|
|
1559
|
+
});
|
|
1560
|
+
const args = {
|
|
1561
|
+
project_name: projectName,
|
|
1562
|
+
project_id: projectId,
|
|
1563
|
+
slug,
|
|
1564
|
+
version
|
|
1565
|
+
};
|
|
1566
|
+
const response = await state.logConn().get_json("v1/prompt", args);
|
|
1567
|
+
if (!("objects" in response) || response.objects.length === 0) {
|
|
1568
|
+
throw new Error(
|
|
1569
|
+
`Prompt ${slug} not found in ${[projectName ?? projectId]}`
|
|
1570
|
+
);
|
|
1571
|
+
} else if (response.objects.length > 1) {
|
|
1572
|
+
throw new Error(
|
|
1573
|
+
`Multiple prompts found with slug ${slug} in project ${projectName ?? projectId}. This should never happen.`
|
|
1574
|
+
);
|
|
1575
|
+
}
|
|
1576
|
+
const metadata = promptSchema.parse(response["objects"][0]);
|
|
1577
|
+
return new Prompt(metadata, defaults || {}, noTrace);
|
|
1578
|
+
}
|
|
1579
|
+
async function login(options = {}) {
|
|
1580
|
+
let { forceLogin = false } = options || {};
|
|
1581
|
+
if (_globalState.loggedIn && !forceLogin) {
|
|
1582
|
+
let checkUpdatedParam2 = function(varname, arg, orig) {
|
|
1583
|
+
if (!isEmpty(arg) && !isEmpty(orig) && arg !== orig) {
|
|
1584
|
+
throw new Error(
|
|
1585
|
+
`Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
|
|
1586
|
+
);
|
|
1587
|
+
}
|
|
1588
|
+
};
|
|
1589
|
+
var checkUpdatedParam = checkUpdatedParam2;
|
|
1590
|
+
checkUpdatedParam2("appUrl", options.appUrl, _globalState.appUrl);
|
|
1591
|
+
checkUpdatedParam2(
|
|
1592
|
+
"apiKey",
|
|
1593
|
+
options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0,
|
|
1594
|
+
_globalState.loginToken
|
|
1595
|
+
);
|
|
1596
|
+
checkUpdatedParam2("orgName", options.orgName, _globalState.orgName);
|
|
1597
|
+
return _globalState;
|
|
1598
|
+
}
|
|
1599
|
+
await _globalState.login(options);
|
|
1600
|
+
globalThis.__inherited_braintrust_state = _globalState;
|
|
1601
|
+
return _globalState;
|
|
1602
|
+
}
|
|
1603
|
+
async function loginToState(options = {}) {
|
|
1604
|
+
const {
|
|
1605
|
+
appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrust.dev",
|
|
1606
|
+
apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
|
|
1607
|
+
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME"),
|
|
1608
|
+
fetch = globalThis.fetch
|
|
1609
|
+
} = options || {};
|
|
1610
|
+
const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
|
|
1611
|
+
const state = new BraintrustState(options);
|
|
1612
|
+
state.resetLoginInfo();
|
|
1613
|
+
state.appUrl = appUrl;
|
|
1614
|
+
state.appPublicUrl = appPublicUrl;
|
|
1615
|
+
let conn = null;
|
|
1616
|
+
if (apiKey !== void 0) {
|
|
1617
|
+
const resp = await checkResponse(
|
|
1618
|
+
await fetch(_urljoin(state.appUrl, `/api/apikey/login`), {
|
|
1619
|
+
method: "POST",
|
|
1620
|
+
headers: {
|
|
1621
|
+
"Content-Type": "application/json",
|
|
1622
|
+
Authorization: `Bearer ${apiKey}`
|
|
1623
|
+
}
|
|
1624
|
+
})
|
|
1625
|
+
);
|
|
1626
|
+
const info = await resp.json();
|
|
1627
|
+
_check_org_info(state, info.org_info, orgName);
|
|
1628
|
+
conn = state.logConn();
|
|
1629
|
+
conn.set_token(apiKey);
|
|
1630
|
+
} else {
|
|
1631
|
+
throw new Error(
|
|
1632
|
+
"Please specify an api key (e.g. by setting BRAINTRUST_API_KEY)."
|
|
1633
|
+
);
|
|
1634
|
+
}
|
|
1635
|
+
if (!conn) {
|
|
1636
|
+
throw new Error("Conn should be set at this point (a bug)");
|
|
1637
|
+
}
|
|
1638
|
+
conn.make_long_lived();
|
|
1639
|
+
state.apiConn().set_token(apiKey);
|
|
1640
|
+
state.proxyConn().set_token(apiKey);
|
|
1641
|
+
state.loginToken = conn.token;
|
|
1642
|
+
state.loggedIn = true;
|
|
1643
|
+
state.loginReplaceLogConn(conn);
|
|
1644
|
+
return state;
|
|
1645
|
+
}
|
|
1646
|
+
function log(event) {
|
|
1647
|
+
console.warn(
|
|
1648
|
+
"braintrust.log is deprecated and will be removed in a future version of braintrust. Use `experiment.log` instead."
|
|
1649
|
+
);
|
|
1650
|
+
const e = currentExperiment();
|
|
1651
|
+
if (!e) {
|
|
1652
|
+
throw new Error("Not initialized. Please call init() first");
|
|
1653
|
+
}
|
|
1654
|
+
return e.log(event);
|
|
1655
|
+
}
|
|
1656
|
+
async function summarize(options = {}) {
|
|
1657
|
+
console.warn(
|
|
1658
|
+
"braintrust.summarize is deprecated and will be removed in a future version of braintrust. Use `experiment.summarize` instead."
|
|
1659
|
+
);
|
|
1660
|
+
const e = currentExperiment();
|
|
1661
|
+
if (!e) {
|
|
1662
|
+
throw new Error("Not initialized. Please call init() first");
|
|
1663
|
+
}
|
|
1664
|
+
return await e.summarize(options);
|
|
1665
|
+
}
|
|
1666
|
+
function currentExperiment(options) {
|
|
1667
|
+
const state = options?.state ?? _globalState;
|
|
1668
|
+
return state.currentExperiment;
|
|
1669
|
+
}
|
|
1670
|
+
function currentLogger(options) {
|
|
1671
|
+
const state = options?.state ?? _globalState;
|
|
1672
|
+
return castLogger(state.currentLogger, options?.asyncFlush);
|
|
1673
|
+
}
|
|
1674
|
+
function currentSpan(options) {
|
|
1675
|
+
const state = options?.state ?? _globalState;
|
|
1676
|
+
return state.currentSpan.getStore() ?? NOOP_SPAN;
|
|
1677
|
+
}
|
|
1678
|
+
function getSpanParentObject(options) {
|
|
1679
|
+
const state = options?.state ?? _globalState;
|
|
1680
|
+
const parentSpan = currentSpan({ state });
|
|
1681
|
+
if (!Object.is(parentSpan, NOOP_SPAN)) {
|
|
1682
|
+
return parentSpan;
|
|
1683
|
+
}
|
|
1684
|
+
const experiment = currentExperiment();
|
|
1685
|
+
if (experiment) {
|
|
1686
|
+
return experiment;
|
|
1687
|
+
}
|
|
1688
|
+
const logger = currentLogger(options);
|
|
1689
|
+
if (logger) {
|
|
1690
|
+
return logger;
|
|
1691
|
+
}
|
|
1692
|
+
return NOOP_SPAN;
|
|
1693
|
+
}
|
|
1694
|
+
function traced(callback, args) {
|
|
1695
|
+
const { span, isLogger } = startSpanAndIsLogger(args);
|
|
1696
|
+
const ret = runFinally(
|
|
1697
|
+
() => {
|
|
1698
|
+
if (args?.setCurrent ?? true) {
|
|
1699
|
+
return withCurrent(span, callback);
|
|
1700
|
+
} else {
|
|
1701
|
+
return callback(span);
|
|
1702
|
+
}
|
|
1703
|
+
},
|
|
1704
|
+
() => span.end()
|
|
1705
|
+
);
|
|
1706
|
+
if (args?.asyncFlush) {
|
|
1707
|
+
return ret;
|
|
1708
|
+
} else {
|
|
1709
|
+
return (async () => {
|
|
1710
|
+
const awaitedRet = await ret;
|
|
1711
|
+
if (isLogger) {
|
|
1712
|
+
await span.flush();
|
|
1713
|
+
}
|
|
1714
|
+
return awaitedRet;
|
|
1715
|
+
})();
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
function wrapTraced(fn, args) {
|
|
1719
|
+
const spanArgs = {
|
|
1720
|
+
name: fn.name,
|
|
1721
|
+
type: "function",
|
|
1722
|
+
...args
|
|
1723
|
+
};
|
|
1724
|
+
const hasExplicitInput = args && args.event && "input" in args.event && args.event.input !== void 0;
|
|
1725
|
+
const hasExplicitOutput = args && args.event && args.event.output !== void 0;
|
|
1726
|
+
if (args?.asyncFlush) {
|
|
1727
|
+
return (...fnArgs) => traced((span) => {
|
|
1728
|
+
if (!hasExplicitInput) {
|
|
1729
|
+
span.log({ input: fnArgs });
|
|
1730
|
+
}
|
|
1731
|
+
const output = fn(...fnArgs);
|
|
1732
|
+
if (!hasExplicitOutput) {
|
|
1733
|
+
if (output instanceof Promise) {
|
|
1734
|
+
return (async () => {
|
|
1735
|
+
const result = await output;
|
|
1736
|
+
span.log({ output: result });
|
|
1737
|
+
return result;
|
|
1738
|
+
})();
|
|
1739
|
+
} else {
|
|
1740
|
+
span.log({ output });
|
|
1741
|
+
}
|
|
1742
|
+
}
|
|
1743
|
+
return output;
|
|
1744
|
+
}, spanArgs);
|
|
1745
|
+
} else {
|
|
1746
|
+
return (...fnArgs) => traced(async (span) => {
|
|
1747
|
+
if (!hasExplicitInput) {
|
|
1748
|
+
span.log({ input: fnArgs });
|
|
1749
|
+
}
|
|
1750
|
+
const outputResult = fn(...fnArgs);
|
|
1751
|
+
const output = await outputResult;
|
|
1752
|
+
if (!hasExplicitOutput) {
|
|
1753
|
+
span.log({ output });
|
|
1754
|
+
}
|
|
1755
|
+
return output;
|
|
1756
|
+
}, spanArgs);
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
var traceable = wrapTraced;
|
|
1760
|
+
function startSpan(args) {
|
|
1761
|
+
return startSpanAndIsLogger(args).span;
|
|
1762
|
+
}
|
|
1763
|
+
async function flush(options) {
|
|
1764
|
+
const state = options?.state ?? _globalState;
|
|
1765
|
+
return await state.bgLogger().flush();
|
|
1766
|
+
}
|
|
1767
|
+
function setFetch(fetch) {
|
|
1768
|
+
_globalState.setFetch(fetch);
|
|
1769
|
+
}
|
|
1770
|
+
function startSpanAndIsLogger(args) {
|
|
1771
|
+
const state = args?.state ?? _globalState;
|
|
1772
|
+
if (args?.parent) {
|
|
1773
|
+
const components = SpanComponentsV2.fromStr(args?.parent);
|
|
1774
|
+
const parentSpanIds = components.rowIds ? {
|
|
1775
|
+
spanId: components.rowIds.spanId,
|
|
1776
|
+
rootSpanId: components.rowIds.rootSpanId
|
|
1777
|
+
} : void 0;
|
|
1778
|
+
const span = new SpanImpl({
|
|
1779
|
+
state,
|
|
1780
|
+
...args,
|
|
1781
|
+
parentObjectType: components.objectType,
|
|
1782
|
+
parentObjectId: new LazyValue(
|
|
1783
|
+
spanComponentsToObjectIdLambda(state, components)
|
|
1784
|
+
),
|
|
1785
|
+
parentComputeObjectMetadataArgs: components.computeObjectMetadataArgs,
|
|
1786
|
+
parentSpanIds
|
|
1787
|
+
});
|
|
1788
|
+
return {
|
|
1789
|
+
span,
|
|
1790
|
+
isLogger: components.objectType === SpanObjectTypeV2.PROJECT_LOGS
|
|
1791
|
+
};
|
|
1792
|
+
} else {
|
|
1793
|
+
const parentObject = getSpanParentObject({
|
|
1794
|
+
asyncFlush: args?.asyncFlush
|
|
1795
|
+
});
|
|
1796
|
+
const span = parentObject.startSpan(args);
|
|
1797
|
+
return { span, isLogger: parentObject.kind === "logger" };
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
function withCurrent(span, callback, state = _globalState) {
|
|
1801
|
+
return state.currentSpan.run(span, () => callback(span));
|
|
1802
|
+
}
|
|
1803
|
+
function _check_org_info(state, org_info, org_name) {
|
|
1804
|
+
if (org_info.length === 0) {
|
|
1805
|
+
throw new Error("This user is not part of any organizations.");
|
|
1806
|
+
}
|
|
1807
|
+
for (const org of org_info) {
|
|
1808
|
+
if (org_name === void 0 || org.name === org_name) {
|
|
1809
|
+
state.orgId = org.id;
|
|
1810
|
+
state.orgName = org.name;
|
|
1811
|
+
state.logUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") ?? org.api_url;
|
|
1812
|
+
state.proxyUrl = isomorph_default.getEnv("BRAINTRUST_PROXY_URL") ?? org.proxy_url;
|
|
1813
|
+
if (state.proxyUrl) {
|
|
1814
|
+
const url = new URL(state.proxyUrl);
|
|
1815
|
+
url.pathname = "";
|
|
1816
|
+
state.proxyUrl = url.toString();
|
|
1817
|
+
}
|
|
1818
|
+
state.gitMetadataSettings = org.git_metadata || void 0;
|
|
1819
|
+
break;
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
if (state.orgId === void 0) {
|
|
1823
|
+
throw new Error(
|
|
1824
|
+
`Organization ${org_name} not found. Must be one of ${org_info.map((x) => x.name).join(", ")}`
|
|
1825
|
+
);
|
|
1826
|
+
}
|
|
1827
|
+
}
|
|
1828
|
+
function _urljoin(...parts) {
|
|
1829
|
+
return parts.map(
|
|
1830
|
+
(x, i) => x.replace(/^\//, "").replace(i < parts.length - 1 ? /\/$/ : "", "")
|
|
1831
|
+
).join("/");
|
|
1832
|
+
}
|
|
1833
|
+
function validateTags(tags) {
|
|
1834
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1835
|
+
for (const tag of tags) {
|
|
1836
|
+
if (typeof tag !== "string") {
|
|
1837
|
+
throw new Error("tags must be strings");
|
|
1838
|
+
}
|
|
1839
|
+
if (seen.has(tag)) {
|
|
1840
|
+
throw new Error(`duplicate tag: ${tag}`);
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
function validateAndSanitizeExperimentLogPartialArgs(event) {
|
|
1845
|
+
if (event.scores) {
|
|
1846
|
+
if (Array.isArray(event.scores)) {
|
|
1847
|
+
throw new Error("scores must be an object, not an array");
|
|
1848
|
+
}
|
|
1849
|
+
for (let [name, score] of Object.entries(event.scores)) {
|
|
1850
|
+
if (typeof name !== "string") {
|
|
1851
|
+
throw new Error("score names must be strings");
|
|
1852
|
+
}
|
|
1853
|
+
if (score === null || score === void 0) {
|
|
1854
|
+
continue;
|
|
1855
|
+
}
|
|
1856
|
+
if (typeof score === "boolean") {
|
|
1857
|
+
score = score ? 1 : 0;
|
|
1858
|
+
event.scores[name] = score;
|
|
1859
|
+
}
|
|
1860
|
+
if (typeof score !== "number") {
|
|
1861
|
+
throw new Error("score values must be numbers");
|
|
1862
|
+
}
|
|
1863
|
+
if (score < 0 || score > 1) {
|
|
1864
|
+
throw new Error("score values must be between 0 and 1");
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
if (event.metadata) {
|
|
1869
|
+
for (const key of Object.keys(event.metadata)) {
|
|
1870
|
+
if (typeof key !== "string") {
|
|
1871
|
+
throw new Error("metadata keys must be strings");
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
if (event.metrics) {
|
|
1876
|
+
for (const [key, value] of Object.entries(event.metrics)) {
|
|
1877
|
+
if (typeof key !== "string") {
|
|
1878
|
+
throw new Error("metric keys must be strings");
|
|
1879
|
+
}
|
|
1880
|
+
if (value !== void 0 && typeof value !== "number") {
|
|
1881
|
+
throw new Error("metric values must be numbers");
|
|
1882
|
+
}
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
if ("input" in event && event.input && "inputs" in event && event.inputs) {
|
|
1886
|
+
throw new Error(
|
|
1887
|
+
"Only one of input or inputs (deprecated) can be specified. Prefer input."
|
|
1888
|
+
);
|
|
1889
|
+
}
|
|
1890
|
+
if ("tags" in event && event.tags) {
|
|
1891
|
+
validateTags(event.tags);
|
|
1892
|
+
}
|
|
1893
|
+
if ("inputs" in event) {
|
|
1894
|
+
const { inputs, ...rest } = event;
|
|
1895
|
+
return { input: inputs, ...rest };
|
|
1896
|
+
} else {
|
|
1897
|
+
return { ...event };
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
1901
|
+
if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
|
|
1902
|
+
throw new Error(
|
|
1903
|
+
"Exactly one of input or inputs (deprecated) must be specified. Prefer input."
|
|
1904
|
+
);
|
|
1905
|
+
}
|
|
1906
|
+
if (isEmpty(event.output)) {
|
|
1907
|
+
throw new Error("output must be specified");
|
|
1908
|
+
}
|
|
1909
|
+
if (isEmpty(event.scores)) {
|
|
1910
|
+
throw new Error("scores must be specified");
|
|
1911
|
+
}
|
|
1912
|
+
if (hasDataset && event.datasetRecordId === void 0) {
|
|
1913
|
+
throw new Error("datasetRecordId must be specified when using a dataset");
|
|
1914
|
+
} else if (!hasDataset && event.datasetRecordId !== void 0) {
|
|
1915
|
+
throw new Error(
|
|
1916
|
+
"datasetRecordId cannot be specified when not using a dataset"
|
|
1917
|
+
);
|
|
1918
|
+
}
|
|
1919
|
+
return event;
|
|
1920
|
+
}
|
|
1921
|
+
var ObjectFetcher = class {
|
|
1922
|
+
constructor(objectType, pinnedVersion, mutateRecord) {
|
|
1923
|
+
this.objectType = objectType;
|
|
1924
|
+
this.pinnedVersion = pinnedVersion;
|
|
1925
|
+
this.mutateRecord = mutateRecord;
|
|
1926
|
+
}
|
|
1927
|
+
_fetchedData = void 0;
|
|
1928
|
+
get id() {
|
|
1929
|
+
throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
|
|
1930
|
+
}
|
|
1931
|
+
async getState() {
|
|
1932
|
+
throw new Error("ObjectFetcher subclasses must have a 'getState' method");
|
|
1933
|
+
}
|
|
1934
|
+
async *fetch() {
|
|
1935
|
+
const records = await this.fetchedData();
|
|
1936
|
+
for (const record of records) {
|
|
1937
|
+
yield record;
|
|
1938
|
+
}
|
|
1939
|
+
}
|
|
1940
|
+
[Symbol.asyncIterator]() {
|
|
1941
|
+
return this.fetch();
|
|
1942
|
+
}
|
|
1943
|
+
async fetchedData() {
|
|
1944
|
+
if (this._fetchedData === void 0) {
|
|
1945
|
+
const state = await this.getState();
|
|
1946
|
+
const resp = await state.logConn().get(
|
|
1947
|
+
`v1/${this.objectType}/${await this.id}/fetch`,
|
|
1948
|
+
{
|
|
1949
|
+
version: this.pinnedVersion
|
|
1950
|
+
},
|
|
1951
|
+
{ headers: { "Accept-Encoding": "gzip" } }
|
|
1952
|
+
);
|
|
1953
|
+
const data = (await resp.json()).events;
|
|
1954
|
+
this._fetchedData = this.mutateRecord ? data?.map(this.mutateRecord) : data;
|
|
1955
|
+
}
|
|
1956
|
+
return this._fetchedData || [];
|
|
1957
|
+
}
|
|
1958
|
+
clearCache() {
|
|
1959
|
+
this._fetchedData = void 0;
|
|
1960
|
+
}
|
|
1961
|
+
async version() {
|
|
1962
|
+
if (this.pinnedVersion !== void 0) {
|
|
1963
|
+
return this.pinnedVersion;
|
|
1964
|
+
} else {
|
|
1965
|
+
const fetchedData = await this.fetchedData();
|
|
1966
|
+
let maxVersion = void 0;
|
|
1967
|
+
for (const record of fetchedData) {
|
|
1968
|
+
const xactId = String(record[TRANSACTION_ID_FIELD] ?? "0");
|
|
1969
|
+
if (maxVersion === void 0 || xactId > maxVersion) {
|
|
1970
|
+
maxVersion = xactId;
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
return maxVersion;
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
};
|
|
1977
|
+
var Experiment = class extends ObjectFetcher {
|
|
1978
|
+
lazyMetadata;
|
|
1979
|
+
dataset;
|
|
1980
|
+
lastStartTime;
|
|
1981
|
+
lazyId;
|
|
1982
|
+
calledStartSpan;
|
|
1983
|
+
state;
|
|
1984
|
+
// For type identification.
|
|
1985
|
+
kind = "experiment";
|
|
1986
|
+
constructor(state, lazyMetadata, dataset) {
|
|
1987
|
+
super("experiment", void 0);
|
|
1988
|
+
this.lazyMetadata = lazyMetadata;
|
|
1989
|
+
this.dataset = dataset;
|
|
1990
|
+
this.lastStartTime = getCurrentUnixTimestamp();
|
|
1991
|
+
this.lazyId = new LazyValue(async () => await this.id);
|
|
1992
|
+
this.calledStartSpan = false;
|
|
1993
|
+
this.state = state;
|
|
1994
|
+
}
|
|
1995
|
+
get id() {
|
|
1996
|
+
return (async () => {
|
|
1997
|
+
return (await this.lazyMetadata.get()).experiment.id;
|
|
1998
|
+
})();
|
|
1999
|
+
}
|
|
2000
|
+
get name() {
|
|
2001
|
+
return (async () => {
|
|
2002
|
+
return (await this.lazyMetadata.get()).experiment.name;
|
|
2003
|
+
})();
|
|
2004
|
+
}
|
|
2005
|
+
get project() {
|
|
2006
|
+
return (async () => {
|
|
2007
|
+
return (await this.lazyMetadata.get()).project;
|
|
2008
|
+
})();
|
|
2009
|
+
}
|
|
2010
|
+
parentObjectType() {
|
|
2011
|
+
return SpanObjectTypeV2.EXPERIMENT;
|
|
2012
|
+
}
|
|
2013
|
+
async getState() {
|
|
2014
|
+
await this.lazyMetadata.get();
|
|
2015
|
+
return this.state;
|
|
2016
|
+
}
|
|
2017
|
+
/**
|
|
2018
|
+
* Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
|
|
2019
|
+
*
|
|
2020
|
+
* @param event The event to log.
|
|
2021
|
+
* @param event.input: The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical.
|
|
2022
|
+
* @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
|
|
2023
|
+
* @param event.expected: (Optional) The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
2024
|
+
* @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
|
|
2025
|
+
* @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
|
|
2026
|
+
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
|
|
2027
|
+
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
2028
|
+
* @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
|
|
2029
|
+
* @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
|
|
2030
|
+
* @param options Additional logging options
|
|
2031
|
+
* @param options.allowConcurrentWithSpans in rare cases where you need to log at the top level separately from spans on the experiment elsewhere, set this to true.
|
|
2032
|
+
* :returns: The `id` of the logged event.
|
|
2033
|
+
*/
|
|
2034
|
+
log(event, options) {
|
|
2035
|
+
if (this.calledStartSpan && !options?.allowConcurrentWithSpans) {
|
|
2036
|
+
throw new Error(
|
|
2037
|
+
"Cannot run toplevel `log` method while using spans. To log to the span, call `experiment.traced` and then log with `span.log`"
|
|
2038
|
+
);
|
|
2039
|
+
}
|
|
2040
|
+
event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
|
|
2041
|
+
const span = this.startSpanImpl({ startTime: this.lastStartTime, event });
|
|
2042
|
+
this.lastStartTime = span.end();
|
|
2043
|
+
return span.id;
|
|
2044
|
+
}
|
|
2045
|
+
/**
|
|
2046
|
+
* Create a new toplevel span underneath the experiment. The name defaults to "root".
|
|
2047
|
+
*
|
|
2048
|
+
* See `Span.traced` for full details.
|
|
2049
|
+
*/
|
|
2050
|
+
traced(callback, args) {
|
|
2051
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
2052
|
+
const span = this.startSpan(argsRest);
|
|
2053
|
+
return runFinally(
|
|
2054
|
+
() => {
|
|
2055
|
+
if (setCurrent ?? true) {
|
|
2056
|
+
return withCurrent(span, callback);
|
|
2057
|
+
} else {
|
|
2058
|
+
return callback(span);
|
|
2059
|
+
}
|
|
2060
|
+
},
|
|
2061
|
+
() => span.end()
|
|
2062
|
+
);
|
|
2063
|
+
}
|
|
2064
|
+
/**
|
|
2065
|
+
* Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
|
|
2066
|
+
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
2067
|
+
* so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
|
|
2068
|
+
*
|
|
2069
|
+
* See `traced` for full details.
|
|
2070
|
+
*/
|
|
2071
|
+
startSpan(args) {
|
|
2072
|
+
this.calledStartSpan = true;
|
|
2073
|
+
return this.startSpanImpl(args);
|
|
2074
|
+
}
|
|
2075
|
+
startSpanImpl(args) {
|
|
2076
|
+
return new SpanImpl({
|
|
2077
|
+
state: this.state,
|
|
2078
|
+
...startSpanParentArgs({
|
|
2079
|
+
state: this.state,
|
|
2080
|
+
parent: args?.parent,
|
|
2081
|
+
parentObjectType: this.parentObjectType(),
|
|
2082
|
+
parentObjectId: this.lazyId,
|
|
2083
|
+
parentComputeObjectMetadataArgs: void 0,
|
|
2084
|
+
parentSpanIds: void 0
|
|
2085
|
+
}),
|
|
2086
|
+
...args,
|
|
2087
|
+
defaultRootType: SpanTypeAttribute.EVAL
|
|
2088
|
+
});
|
|
2089
|
+
}
|
|
2090
|
+
async fetchBaseExperiment() {
|
|
2091
|
+
const state = await this.getState();
|
|
2092
|
+
const conn = state.apiConn();
|
|
2093
|
+
try {
|
|
2094
|
+
const resp = await conn.post("/api/base_experiment/get_id", {
|
|
2095
|
+
id: await this.id
|
|
2096
|
+
});
|
|
2097
|
+
const base = await resp.json();
|
|
2098
|
+
return {
|
|
2099
|
+
id: base["base_exp_id"],
|
|
2100
|
+
name: base["base_exp_name"]
|
|
2101
|
+
};
|
|
2102
|
+
} catch (e) {
|
|
2103
|
+
if (e instanceof FailedHTTPResponse && e.status === 400) {
|
|
2104
|
+
return null;
|
|
2105
|
+
} else {
|
|
2106
|
+
throw e;
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
2109
|
+
}
|
|
2110
|
+
/**
|
|
2111
|
+
* Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
2112
|
+
*
|
|
2113
|
+
* @param options Options for summarizing the experiment.
|
|
2114
|
+
* @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
|
|
2115
|
+
* @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
|
|
2116
|
+
* @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
2117
|
+
*/
|
|
2118
|
+
async summarize(options = {}) {
|
|
2119
|
+
let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
|
|
2120
|
+
await this.flush();
|
|
2121
|
+
const state = await this.getState();
|
|
2122
|
+
const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
|
|
2123
|
+
state.orgName
|
|
2124
|
+
)}/p/${encodeURIComponent((await this.project).name)}`;
|
|
2125
|
+
const experimentUrl = `${projectUrl}/experiments/${encodeURIComponent(
|
|
2126
|
+
await this.name
|
|
2127
|
+
)}`;
|
|
2128
|
+
let scores = void 0;
|
|
2129
|
+
let metrics = void 0;
|
|
2130
|
+
let comparisonExperimentName = void 0;
|
|
2131
|
+
if (summarizeScores) {
|
|
2132
|
+
if (comparisonExperimentId === void 0) {
|
|
2133
|
+
const baseExperiment = await this.fetchBaseExperiment();
|
|
2134
|
+
if (baseExperiment !== null) {
|
|
2135
|
+
comparisonExperimentId = baseExperiment.id;
|
|
2136
|
+
comparisonExperimentName = baseExperiment.name;
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
const results = await state.logConn().get_json(
|
|
2140
|
+
"/experiment-comparison2",
|
|
2141
|
+
{
|
|
2142
|
+
experiment_id: await this.id,
|
|
2143
|
+
base_experiment_id: comparisonExperimentId
|
|
2144
|
+
},
|
|
2145
|
+
3
|
|
2146
|
+
);
|
|
2147
|
+
scores = results["scores"];
|
|
2148
|
+
metrics = results["metrics"];
|
|
2149
|
+
}
|
|
2150
|
+
return {
|
|
2151
|
+
projectName: (await this.project).name,
|
|
2152
|
+
experimentName: await this.name,
|
|
2153
|
+
projectId: (await this.project).id,
|
|
2154
|
+
experimentId: await this.id,
|
|
2155
|
+
projectUrl,
|
|
2156
|
+
experimentUrl,
|
|
2157
|
+
comparisonExperimentName,
|
|
2158
|
+
scores: scores ?? {},
|
|
2159
|
+
metrics
|
|
2160
|
+
};
|
|
2161
|
+
}
|
|
2162
|
+
/**
|
|
2163
|
+
* Log feedback to an event in the experiment. Feedback is used to save feedback scores, set an expected value, or add a comment.
|
|
2164
|
+
*
|
|
2165
|
+
* @param event
|
|
2166
|
+
* @param event.id The id of the event to log feedback for. This is the `id` returned by `log` or accessible as the `id` field of a span.
|
|
2167
|
+
* @param event.scores (Optional) a dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the event.
|
|
2168
|
+
* @param event.expected (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not.
|
|
2169
|
+
* @param event.comment (Optional) an optional comment string to log about the event.
|
|
2170
|
+
* @param event.metadata (Optional) a dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI.
|
|
2171
|
+
* @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
|
|
2172
|
+
*/
|
|
2173
|
+
logFeedback(event) {
|
|
2174
|
+
logFeedbackImpl(this.state, this.parentObjectType(), this.lazyId, event);
|
|
2175
|
+
}
|
|
2176
|
+
/**
|
|
2177
|
+
* Return a serialized representation of the experiment that can be used to start subspans in other places. See `Span.start_span` for more details.
|
|
2178
|
+
*/
|
|
2179
|
+
async export() {
|
|
2180
|
+
return new SpanComponentsV2({
|
|
2181
|
+
objectType: this.parentObjectType(),
|
|
2182
|
+
objectId: await this.id
|
|
2183
|
+
}).toStr();
|
|
2184
|
+
}
|
|
2185
|
+
/**
|
|
2186
|
+
* Flush any pending rows to the server.
|
|
2187
|
+
*/
|
|
2188
|
+
async flush() {
|
|
2189
|
+
return await this.state.bgLogger().flush();
|
|
2190
|
+
}
|
|
2191
|
+
/**
|
|
2192
|
+
* This function is deprecated. You can simply remove it from your code.
|
|
2193
|
+
*/
|
|
2194
|
+
async close() {
|
|
2195
|
+
console.warn(
|
|
2196
|
+
"close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
|
|
2197
|
+
);
|
|
2198
|
+
return this.id;
|
|
2199
|
+
}
|
|
2200
|
+
};
|
|
2201
|
+
var ReadonlyExperiment = class extends ObjectFetcher {
|
|
2202
|
+
constructor(state, lazyMetadata) {
|
|
2203
|
+
super("experiment", void 0);
|
|
2204
|
+
this.state = state;
|
|
2205
|
+
this.lazyMetadata = lazyMetadata;
|
|
2206
|
+
}
|
|
2207
|
+
get id() {
|
|
2208
|
+
return (async () => {
|
|
2209
|
+
return (await this.lazyMetadata.get()).experiment.id;
|
|
2210
|
+
})();
|
|
2211
|
+
}
|
|
2212
|
+
get name() {
|
|
2213
|
+
return (async () => {
|
|
2214
|
+
return (await this.lazyMetadata.get()).experiment.name;
|
|
2215
|
+
})();
|
|
2216
|
+
}
|
|
2217
|
+
async getState() {
|
|
2218
|
+
await this.lazyMetadata.get();
|
|
2219
|
+
return this.state;
|
|
2220
|
+
}
|
|
2221
|
+
async *asDataset() {
|
|
2222
|
+
const records = this.fetch();
|
|
2223
|
+
for await (const record of records) {
|
|
2224
|
+
if (record.root_span_id !== record.span_id) {
|
|
2225
|
+
continue;
|
|
2226
|
+
}
|
|
2227
|
+
const { output, expected: expectedRecord } = record;
|
|
2228
|
+
const expected = expectedRecord ?? output;
|
|
2229
|
+
if (isEmpty(expected)) {
|
|
2230
|
+
yield {
|
|
2231
|
+
input: record.input,
|
|
2232
|
+
tags: record.tags
|
|
2233
|
+
};
|
|
2234
|
+
} else {
|
|
2235
|
+
yield {
|
|
2236
|
+
input: record.input,
|
|
2237
|
+
expected,
|
|
2238
|
+
tags: record.tags
|
|
2239
|
+
};
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
};
|
|
2244
|
+
var executionCounter = 0;
|
|
2245
|
+
function newId() {
|
|
2246
|
+
return uuidv4();
|
|
2247
|
+
}
|
|
2248
|
+
var SpanImpl = class _SpanImpl {
|
|
2249
|
+
state;
|
|
2250
|
+
// `internalData` contains fields that are not part of the "user-sanitized"
|
|
2251
|
+
// set of fields which we want to log in just one of the span rows.
|
|
2252
|
+
isMerge;
|
|
2253
|
+
loggedEndTime;
|
|
2254
|
+
// For internal use only.
|
|
2255
|
+
parentObjectType;
|
|
2256
|
+
parentObjectId;
|
|
2257
|
+
parentComputeObjectMetadataArgs;
|
|
2258
|
+
_id;
|
|
2259
|
+
spanId;
|
|
2260
|
+
rootSpanId;
|
|
2261
|
+
spanParents;
|
|
2262
|
+
kind = "span";
|
|
2263
|
+
constructor(args) {
|
|
2264
|
+
this.state = args.state;
|
|
2265
|
+
const spanAttributes = args.spanAttributes ?? {};
|
|
2266
|
+
const event = args.event ?? {};
|
|
2267
|
+
const type = args.type ?? (args.parentSpanIds ? void 0 : args.defaultRootType);
|
|
2268
|
+
this.loggedEndTime = void 0;
|
|
2269
|
+
this.parentObjectType = args.parentObjectType;
|
|
2270
|
+
this.parentObjectId = args.parentObjectId;
|
|
2271
|
+
this.parentComputeObjectMetadataArgs = args.parentComputeObjectMetadataArgs;
|
|
2272
|
+
const callerLocation = isomorph_default.getCallerLocation();
|
|
2273
|
+
const name = (() => {
|
|
2274
|
+
if (args.name)
|
|
2275
|
+
return args.name;
|
|
2276
|
+
if (!args.parentSpanIds)
|
|
2277
|
+
return "root";
|
|
2278
|
+
if (callerLocation) {
|
|
2279
|
+
const pathComponents = callerLocation.caller_filename.split("/");
|
|
2280
|
+
const filename = pathComponents[pathComponents.length - 1];
|
|
2281
|
+
return [callerLocation.caller_functionname].concat(
|
|
2282
|
+
filename ? [`${filename}:${callerLocation.caller_lineno}`] : []
|
|
2283
|
+
).join(":");
|
|
2284
|
+
}
|
|
2285
|
+
return "subspan";
|
|
2286
|
+
})();
|
|
2287
|
+
const internalData = {
|
|
2288
|
+
metrics: {
|
|
2289
|
+
start: args.startTime ?? getCurrentUnixTimestamp()
|
|
2290
|
+
},
|
|
2291
|
+
context: { ...callerLocation },
|
|
2292
|
+
span_attributes: {
|
|
2293
|
+
name,
|
|
2294
|
+
type,
|
|
2295
|
+
...spanAttributes,
|
|
2296
|
+
exec_counter: executionCounter++
|
|
2297
|
+
},
|
|
2298
|
+
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
2299
|
+
};
|
|
2300
|
+
this._id = event.id ?? uuidv4();
|
|
2301
|
+
this.spanId = uuidv4();
|
|
2302
|
+
if (args.parentSpanIds) {
|
|
2303
|
+
this.rootSpanId = args.parentSpanIds.rootSpanId;
|
|
2304
|
+
this.spanParents = [args.parentSpanIds.spanId];
|
|
2305
|
+
} else {
|
|
2306
|
+
this.rootSpanId = this.spanId;
|
|
2307
|
+
this.spanParents = void 0;
|
|
2308
|
+
}
|
|
2309
|
+
this.isMerge = false;
|
|
2310
|
+
const { id: _id, ...eventRest } = event;
|
|
2311
|
+
this.logInternal({ event: eventRest, internalData });
|
|
2312
|
+
this.isMerge = true;
|
|
2313
|
+
}
|
|
2314
|
+
get id() {
|
|
2315
|
+
return this._id;
|
|
2316
|
+
}
|
|
2317
|
+
setAttributes(args) {
|
|
2318
|
+
this.logInternal({ internalData: { span_attributes: args } });
|
|
2319
|
+
}
|
|
2320
|
+
log(event) {
|
|
2321
|
+
this.logInternal({ event });
|
|
2322
|
+
}
|
|
2323
|
+
logInternal({
|
|
2324
|
+
event,
|
|
2325
|
+
internalData
|
|
2326
|
+
}) {
|
|
2327
|
+
const sanitized = validateAndSanitizeExperimentLogPartialArgs(event ?? {});
|
|
2328
|
+
let sanitizedAndInternalData = { ...internalData };
|
|
2329
|
+
mergeDicts(sanitizedAndInternalData, sanitized);
|
|
2330
|
+
const serializableInternalData = {};
|
|
2331
|
+
const lazyInternalData = {};
|
|
2332
|
+
for (const [key, value] of Object.entries(sanitizedAndInternalData)) {
|
|
2333
|
+
if (value instanceof BraintrustStream) {
|
|
2334
|
+
const streamCopy = value.copy();
|
|
2335
|
+
lazyInternalData[key] = new LazyValue(async () => {
|
|
2336
|
+
return await new Promise((resolve) => {
|
|
2337
|
+
streamCopy.toReadableStream().pipeThrough(createFinalValuePassThroughStream(resolve)).pipeTo(devNullWritableStream());
|
|
2338
|
+
});
|
|
2339
|
+
});
|
|
2340
|
+
} else if (value instanceof ReadableStream) {
|
|
2341
|
+
lazyInternalData[key] = new LazyValue(async () => {
|
|
2342
|
+
return await new Promise((resolve) => {
|
|
2343
|
+
value.pipeThrough(createFinalValuePassThroughStream(resolve)).pipeTo(devNullWritableStream());
|
|
2344
|
+
});
|
|
2345
|
+
});
|
|
2346
|
+
} else {
|
|
2347
|
+
serializableInternalData[key] = value;
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
let partialRecord = {
|
|
2351
|
+
id: this.id,
|
|
2352
|
+
span_id: this.spanId,
|
|
2353
|
+
root_span_id: this.rootSpanId,
|
|
2354
|
+
span_parents: this.spanParents,
|
|
2355
|
+
...serializableInternalData,
|
|
2356
|
+
[IS_MERGE_FIELD]: this.isMerge
|
|
2357
|
+
};
|
|
2358
|
+
const serializedPartialRecord = JSON.stringify(partialRecord, (k, v) => {
|
|
2359
|
+
if (v instanceof _SpanImpl) {
|
|
2360
|
+
return `<span>`;
|
|
2361
|
+
} else if (v instanceof Experiment) {
|
|
2362
|
+
return `<experiment>`;
|
|
2363
|
+
} else if (v instanceof Dataset) {
|
|
2364
|
+
return `<dataset>`;
|
|
2365
|
+
} else if (v instanceof Logger) {
|
|
2366
|
+
return `<logger>`;
|
|
2367
|
+
}
|
|
2368
|
+
return v;
|
|
2369
|
+
});
|
|
2370
|
+
partialRecord = JSON.parse(serializedPartialRecord);
|
|
2371
|
+
if (partialRecord.metrics?.end) {
|
|
2372
|
+
this.loggedEndTime = partialRecord.metrics?.end;
|
|
2373
|
+
}
|
|
2374
|
+
if ((partialRecord.tags ?? []).length > 0 && this.spanParents?.length) {
|
|
2375
|
+
throw new Error("Tags can only be logged to the root span");
|
|
2376
|
+
}
|
|
2377
|
+
const computeRecord = async () => ({
|
|
2378
|
+
...partialRecord,
|
|
2379
|
+
...Object.fromEntries(
|
|
2380
|
+
await Promise.all(
|
|
2381
|
+
Object.entries(lazyInternalData).map(async ([key, value]) => [
|
|
2382
|
+
key,
|
|
2383
|
+
await value.get()
|
|
2384
|
+
])
|
|
2385
|
+
)
|
|
2386
|
+
),
|
|
2387
|
+
...new SpanComponentsV2({
|
|
2388
|
+
objectType: this.parentObjectType,
|
|
2389
|
+
objectId: await this.parentObjectId.get()
|
|
2390
|
+
}).objectIdFields()
|
|
2391
|
+
});
|
|
2392
|
+
this.state.bgLogger().log([new LazyValue(computeRecord)]);
|
|
2393
|
+
}
|
|
2394
|
+
logFeedback(event) {
|
|
2395
|
+
logFeedbackImpl(this.state, this.parentObjectType, this.parentObjectId, {
|
|
2396
|
+
...event,
|
|
2397
|
+
id: this.id
|
|
2398
|
+
});
|
|
2399
|
+
}
|
|
2400
|
+
traced(callback, args) {
|
|
2401
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
2402
|
+
const span = this.startSpan(argsRest);
|
|
2403
|
+
return runFinally(
|
|
2404
|
+
() => {
|
|
2405
|
+
if (setCurrent ?? true) {
|
|
2406
|
+
return withCurrent(span, callback);
|
|
2407
|
+
} else {
|
|
2408
|
+
return callback(span);
|
|
2409
|
+
}
|
|
2410
|
+
},
|
|
2411
|
+
() => span.end()
|
|
2412
|
+
);
|
|
2413
|
+
}
|
|
2414
|
+
startSpan(args) {
|
|
2415
|
+
const parentSpanIds = args?.parent ? void 0 : { spanId: this.spanId, rootSpanId: this.rootSpanId };
|
|
2416
|
+
return new _SpanImpl({
|
|
2417
|
+
state: this.state,
|
|
2418
|
+
...args,
|
|
2419
|
+
...startSpanParentArgs({
|
|
2420
|
+
state: this.state,
|
|
2421
|
+
parent: args?.parent,
|
|
2422
|
+
parentObjectType: this.parentObjectType,
|
|
2423
|
+
parentObjectId: this.parentObjectId,
|
|
2424
|
+
parentComputeObjectMetadataArgs: this.parentComputeObjectMetadataArgs,
|
|
2425
|
+
parentSpanIds
|
|
2426
|
+
})
|
|
2427
|
+
});
|
|
2428
|
+
}
|
|
2429
|
+
end(args) {
|
|
2430
|
+
let endTime;
|
|
2431
|
+
let internalData = {};
|
|
2432
|
+
if (!this.loggedEndTime) {
|
|
2433
|
+
endTime = args?.endTime ?? getCurrentUnixTimestamp();
|
|
2434
|
+
internalData = { metrics: { end: endTime } };
|
|
2435
|
+
} else {
|
|
2436
|
+
endTime = this.loggedEndTime;
|
|
2437
|
+
}
|
|
2438
|
+
this.logInternal({ internalData });
|
|
2439
|
+
return endTime;
|
|
2440
|
+
}
|
|
2441
|
+
async export() {
|
|
2442
|
+
let objectId = void 0;
|
|
2443
|
+
let computeObjectMetadataArgs = void 0;
|
|
2444
|
+
if (this.parentComputeObjectMetadataArgs && !this.parentObjectId.hasComputed) {
|
|
2445
|
+
computeObjectMetadataArgs = this.parentComputeObjectMetadataArgs;
|
|
2446
|
+
} else {
|
|
2447
|
+
objectId = await this.parentObjectId.get();
|
|
2448
|
+
}
|
|
2449
|
+
return new SpanComponentsV2({
|
|
2450
|
+
objectType: this.parentObjectType,
|
|
2451
|
+
objectId,
|
|
2452
|
+
computeObjectMetadataArgs,
|
|
2453
|
+
rowIds: new SpanRowIdsV2({
|
|
2454
|
+
rowId: this.id,
|
|
2455
|
+
spanId: this.spanId,
|
|
2456
|
+
rootSpanId: this.rootSpanId
|
|
2457
|
+
})
|
|
2458
|
+
}).toStr();
|
|
2459
|
+
}
|
|
2460
|
+
async flush() {
|
|
2461
|
+
return await this.state.bgLogger().flush();
|
|
2462
|
+
}
|
|
2463
|
+
close(args) {
|
|
2464
|
+
return this.end(args);
|
|
2465
|
+
}
|
|
2466
|
+
};
|
|
2467
|
+
var Dataset = class extends ObjectFetcher {
|
|
2468
|
+
constructor(state, lazyMetadata, pinnedVersion, legacy) {
|
|
2469
|
+
const isLegacyDataset = legacy ?? DEFAULT_IS_LEGACY_DATASET;
|
|
2470
|
+
if (isLegacyDataset) {
|
|
2471
|
+
console.warn(
|
|
2472
|
+
`Records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". Please update your code to use "expected", and use \`braintrust.initDataset()\` with \`{ useOutput: false }\`, which will become the default in a future version of Braintrust.`
|
|
2473
|
+
);
|
|
2474
|
+
}
|
|
2475
|
+
super(
|
|
2476
|
+
"dataset",
|
|
2477
|
+
pinnedVersion,
|
|
2478
|
+
(r) => ensureDatasetRecord(r, isLegacyDataset)
|
|
2479
|
+
);
|
|
2480
|
+
this.state = state;
|
|
2481
|
+
this.lazyMetadata = lazyMetadata;
|
|
2482
|
+
}
|
|
2483
|
+
lazyMetadata;
|
|
2484
|
+
get id() {
|
|
2485
|
+
return (async () => {
|
|
2486
|
+
return (await this.lazyMetadata.get()).dataset.id;
|
|
2487
|
+
})();
|
|
2488
|
+
}
|
|
2489
|
+
get name() {
|
|
2490
|
+
return (async () => {
|
|
2491
|
+
return (await this.lazyMetadata.get()).dataset.name;
|
|
2492
|
+
})();
|
|
2493
|
+
}
|
|
2494
|
+
get project() {
|
|
2495
|
+
return (async () => {
|
|
2496
|
+
return (await this.lazyMetadata.get()).project;
|
|
2497
|
+
})();
|
|
2498
|
+
}
|
|
2499
|
+
async getState() {
|
|
2500
|
+
await this.lazyMetadata.get();
|
|
2501
|
+
return this.state;
|
|
2502
|
+
}
|
|
2503
|
+
/**
|
|
2504
|
+
* Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
|
|
2505
|
+
* and a record with that `id` already exists, it will be overwritten (upsert).
|
|
2506
|
+
*
|
|
2507
|
+
* @param event The event to log.
|
|
2508
|
+
* @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
|
|
2509
|
+
* @param event.expected The output of your application, including post-processing (an arbitrary, JSON serializable object).
|
|
2510
|
+
* @param event.tags (Optional) a list of strings that you can use to filter and group records later.
|
|
2511
|
+
* @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
|
|
2512
|
+
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
2513
|
+
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
2514
|
+
* JSON-serializable type, but its keys must be strings.
|
|
2515
|
+
* @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
|
|
2516
|
+
* @param event.output: (Deprecated) The output of your application. Use `expected` instead.
|
|
2517
|
+
* @returns The `id` of the logged record.
|
|
2518
|
+
*/
|
|
2519
|
+
insert({
|
|
2520
|
+
input,
|
|
2521
|
+
expected,
|
|
2522
|
+
metadata,
|
|
2523
|
+
tags,
|
|
2524
|
+
id,
|
|
2525
|
+
output
|
|
2526
|
+
}) {
|
|
2527
|
+
if (metadata !== void 0) {
|
|
2528
|
+
for (const key of Object.keys(metadata)) {
|
|
2529
|
+
if (typeof key !== "string") {
|
|
2530
|
+
throw new Error("metadata keys must be strings");
|
|
2531
|
+
}
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
if (expected && output) {
|
|
2535
|
+
throw new Error(
|
|
2536
|
+
"Only one of expected or output (deprecated) can be specified. Prefer expected."
|
|
2537
|
+
);
|
|
2538
|
+
}
|
|
2539
|
+
if (tags) {
|
|
2540
|
+
validateTags(tags);
|
|
2541
|
+
}
|
|
2542
|
+
const rowId = id || uuidv4();
|
|
2543
|
+
const args = new LazyValue(async () => ({
|
|
2544
|
+
id: rowId,
|
|
2545
|
+
input,
|
|
2546
|
+
expected: expected === void 0 ? output : expected,
|
|
2547
|
+
tags,
|
|
2548
|
+
dataset_id: await this.id,
|
|
2549
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2550
|
+
metadata
|
|
2551
|
+
}));
|
|
2552
|
+
this.state.bgLogger().log([args]);
|
|
2553
|
+
return rowId;
|
|
2554
|
+
}
|
|
2555
|
+
delete(id) {
|
|
2556
|
+
const args = new LazyValue(async () => ({
|
|
2557
|
+
id,
|
|
2558
|
+
dataset_id: await this.id,
|
|
2559
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2560
|
+
_object_delete: true
|
|
2561
|
+
}));
|
|
2562
|
+
this.state.bgLogger().log([args]);
|
|
2563
|
+
return id;
|
|
2564
|
+
}
|
|
2565
|
+
/**
|
|
2566
|
+
* Summarize the dataset, including high level metrics about its size and other metadata.
|
|
2567
|
+
* @param summarizeData Whether to summarize the data. If false, only the metadata will be returned.
|
|
2568
|
+
* @returns `DatasetSummary`
|
|
2569
|
+
* @returns A summary of the dataset.
|
|
2570
|
+
*/
|
|
2571
|
+
async summarize(options = {}) {
|
|
2572
|
+
let { summarizeData = true } = options || {};
|
|
2573
|
+
await this.flush();
|
|
2574
|
+
const state = await this.getState();
|
|
2575
|
+
const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
|
|
2576
|
+
state.orgName
|
|
2577
|
+
)}/p/${encodeURIComponent((await this.project).name)}`;
|
|
2578
|
+
const datasetUrl = `${projectUrl}/datasets/${encodeURIComponent(
|
|
2579
|
+
await this.name
|
|
2580
|
+
)}`;
|
|
2581
|
+
let dataSummary = void 0;
|
|
2582
|
+
if (summarizeData) {
|
|
2583
|
+
dataSummary = await state.logConn().get_json(
|
|
2584
|
+
"dataset-summary",
|
|
2585
|
+
{
|
|
2586
|
+
dataset_id: await this.id
|
|
2587
|
+
},
|
|
2588
|
+
3
|
|
2589
|
+
);
|
|
2590
|
+
}
|
|
2591
|
+
return {
|
|
2592
|
+
projectName: (await this.project).name,
|
|
2593
|
+
datasetName: await this.name,
|
|
2594
|
+
projectUrl,
|
|
2595
|
+
datasetUrl,
|
|
2596
|
+
dataSummary
|
|
2597
|
+
};
|
|
2598
|
+
}
|
|
2599
|
+
/**
|
|
2600
|
+
* Flush any pending rows to the server.
|
|
2601
|
+
*/
|
|
2602
|
+
async flush() {
|
|
2603
|
+
return await this.state.bgLogger().flush();
|
|
2604
|
+
}
|
|
2605
|
+
/**
|
|
2606
|
+
* This function is deprecated. You can simply remove it from your code.
|
|
2607
|
+
*/
|
|
2608
|
+
async close() {
|
|
2609
|
+
console.warn(
|
|
2610
|
+
"close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
|
|
2611
|
+
);
|
|
2612
|
+
return this.id;
|
|
2613
|
+
}
|
|
2614
|
+
};
|
|
2615
|
+
var Prompt = class {
|
|
2616
|
+
constructor(metadata, defaults, noTrace) {
|
|
2617
|
+
this.metadata = metadata;
|
|
2618
|
+
this.defaults = defaults;
|
|
2619
|
+
this.noTrace = noTrace;
|
|
2620
|
+
}
|
|
2621
|
+
get id() {
|
|
2622
|
+
return this.metadata.id;
|
|
2623
|
+
}
|
|
2624
|
+
get projectId() {
|
|
2625
|
+
return this.metadata.project_id;
|
|
2626
|
+
}
|
|
2627
|
+
get name() {
|
|
2628
|
+
return "name" in this.metadata ? this.metadata.name : `Playground function ${this.metadata.id}`;
|
|
2629
|
+
}
|
|
2630
|
+
get slug() {
|
|
2631
|
+
return "slug" in this.metadata ? this.metadata.slug : this.metadata.id;
|
|
2632
|
+
}
|
|
2633
|
+
get prompt() {
|
|
2634
|
+
return this.metadata.prompt_data?.prompt;
|
|
2635
|
+
}
|
|
2636
|
+
get version() {
|
|
2637
|
+
return this.metadata[TRANSACTION_ID_FIELD];
|
|
2638
|
+
}
|
|
2639
|
+
get options() {
|
|
2640
|
+
return this.metadata.prompt_data?.options || {};
|
|
2641
|
+
}
|
|
2642
|
+
/**
|
|
2643
|
+
* Build the prompt with the given formatting options. The args you pass in will
|
|
2644
|
+
* be forwarded to the mustache template that defines the prompt and rendered with
|
|
2645
|
+
* the `mustache-js` library.
|
|
2646
|
+
*
|
|
2647
|
+
* @param buildArgs Args to forward along to the prompt template.
|
|
2648
|
+
*/
|
|
2649
|
+
build(buildArgs, options = {}) {
|
|
2650
|
+
return this.runBuild(buildArgs, {
|
|
2651
|
+
flavor: options.flavor ?? "chat"
|
|
2652
|
+
});
|
|
2653
|
+
}
|
|
2654
|
+
runBuild(buildArgs, options) {
|
|
2655
|
+
const { flavor } = options;
|
|
2656
|
+
const params = {
|
|
2657
|
+
...this.defaults,
|
|
2658
|
+
...Object.fromEntries(
|
|
2659
|
+
Object.entries(this.options.params || {}).filter(
|
|
2660
|
+
([k, _v]) => !BRAINTRUST_PARAMS.includes(k)
|
|
2661
|
+
)
|
|
2662
|
+
),
|
|
2663
|
+
...!isEmpty(this.options.model) ? {
|
|
2664
|
+
model: this.options.model
|
|
2665
|
+
} : {}
|
|
2666
|
+
};
|
|
2667
|
+
if (!("model" in params) || isEmpty(params.model)) {
|
|
2668
|
+
throw new Error(
|
|
2669
|
+
"No model specified. Either specify it in the prompt or as a default"
|
|
2670
|
+
);
|
|
2671
|
+
}
|
|
2672
|
+
const spanInfo = this.noTrace ? {} : {
|
|
2673
|
+
span_info: {
|
|
2674
|
+
metadata: {
|
|
2675
|
+
prompt: {
|
|
2676
|
+
variables: buildArgs,
|
|
2677
|
+
id: this.id,
|
|
2678
|
+
project_id: this.projectId,
|
|
2679
|
+
version: this.version,
|
|
2680
|
+
..."prompt_session_id" in this.metadata ? { prompt_session_id: this.metadata.prompt_session_id } : {}
|
|
2681
|
+
}
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
};
|
|
2685
|
+
const prompt = this.prompt;
|
|
2686
|
+
if (!prompt) {
|
|
2687
|
+
throw new Error("Empty prompt");
|
|
2688
|
+
}
|
|
2689
|
+
const dictArgParsed = z.record(z.unknown()).safeParse(buildArgs);
|
|
2690
|
+
const variables = {
|
|
2691
|
+
input: buildArgs,
|
|
2692
|
+
...dictArgParsed.success ? dictArgParsed.data : {}
|
|
2693
|
+
};
|
|
2694
|
+
if (flavor === "chat") {
|
|
2695
|
+
if (prompt.type !== "chat") {
|
|
2696
|
+
throw new Error(
|
|
2697
|
+
"Prompt is a completion prompt. Use buildCompletion() instead"
|
|
2698
|
+
);
|
|
2699
|
+
}
|
|
2700
|
+
const render = (template) => Mustache.render(template, variables, void 0, {
|
|
2701
|
+
escape: (v) => typeof v === "string" ? v : JSON.stringify(v)
|
|
2702
|
+
});
|
|
2703
|
+
const messages = (prompt.messages || []).map((m) => ({
|
|
2704
|
+
...m,
|
|
2705
|
+
..."content" in m ? {
|
|
2706
|
+
content: typeof m.content === "string" ? render(m.content) : JSON.parse(render(JSON.stringify(m.content)))
|
|
2707
|
+
} : {}
|
|
2708
|
+
}));
|
|
2709
|
+
return {
|
|
2710
|
+
...params,
|
|
2711
|
+
...spanInfo,
|
|
2712
|
+
messages,
|
|
2713
|
+
...prompt.tools ? {
|
|
2714
|
+
tools: toolsSchema.parse(
|
|
2715
|
+
JSON.parse(Mustache.render(prompt.tools, variables))
|
|
2716
|
+
)
|
|
2717
|
+
} : void 0
|
|
2718
|
+
};
|
|
2719
|
+
} else if (flavor === "completion") {
|
|
2720
|
+
if (prompt.type !== "completion") {
|
|
2721
|
+
throw new Error("Prompt is a chat prompt. Use buildChat() instead");
|
|
2722
|
+
}
|
|
2723
|
+
return {
|
|
2724
|
+
...params,
|
|
2725
|
+
...spanInfo,
|
|
2726
|
+
prompt: Mustache.render(prompt.content, variables)
|
|
2727
|
+
};
|
|
2728
|
+
} else {
|
|
2729
|
+
throw new Error("never!");
|
|
2730
|
+
}
|
|
2731
|
+
}
|
|
2732
|
+
};
|
|
2733
|
+
|
|
2734
|
+
// src/browser-config.ts
|
|
2735
|
+
var browserConfigured = false;
|
|
2736
|
+
function configureBrowser() {
|
|
2737
|
+
if (browserConfigured) {
|
|
2738
|
+
return;
|
|
2739
|
+
}
|
|
2740
|
+
try {
|
|
2741
|
+
if (typeof AsyncLocalStorage !== "undefined") {
|
|
2742
|
+
isomorph_default.newAsyncLocalStorage = () => new AsyncLocalStorage();
|
|
2743
|
+
}
|
|
2744
|
+
} catch {
|
|
2745
|
+
}
|
|
2746
|
+
isomorph_default.getEnv = (name) => {
|
|
2747
|
+
if (typeof process === "undefined" || typeof process.env === "undefined") {
|
|
2748
|
+
return void 0;
|
|
2749
|
+
}
|
|
2750
|
+
return process.env[name];
|
|
2751
|
+
};
|
|
2752
|
+
_internalSetInitialState();
|
|
2753
|
+
browserConfigured = true;
|
|
2754
|
+
}
|
|
2755
|
+
|
|
2756
|
+
// src/functions/invoke.ts
|
|
2757
|
+
import {
|
|
2758
|
+
INVOKE_API_VERSION
|
|
2759
|
+
} from "@braintrust/core/typespecs";
|
|
2760
|
+
async function invoke(args) {
|
|
2761
|
+
const {
|
|
2762
|
+
orgName,
|
|
2763
|
+
apiKey,
|
|
2764
|
+
appUrl,
|
|
2765
|
+
forceLogin,
|
|
2766
|
+
fetch,
|
|
2767
|
+
arg,
|
|
2768
|
+
parent: parentArg,
|
|
2769
|
+
state: stateArg,
|
|
2770
|
+
stream,
|
|
2771
|
+
schema,
|
|
2772
|
+
...functionId
|
|
2773
|
+
} = args;
|
|
2774
|
+
const state = stateArg ?? _internalGetGlobalState();
|
|
2775
|
+
await state.login({
|
|
2776
|
+
orgName,
|
|
2777
|
+
apiKey,
|
|
2778
|
+
appUrl,
|
|
2779
|
+
forceLogin
|
|
2780
|
+
});
|
|
2781
|
+
const parent = parentArg ? typeof parentArg === "string" ? parentArg : await parentArg.export() : await currentSpan().export();
|
|
2782
|
+
const request = {
|
|
2783
|
+
...functionId,
|
|
2784
|
+
arg,
|
|
2785
|
+
parent,
|
|
2786
|
+
stream,
|
|
2787
|
+
api_version: INVOKE_API_VERSION
|
|
2788
|
+
};
|
|
2789
|
+
const resp = await state.proxyConn().post(`function/invoke`, request, {
|
|
2790
|
+
headers: {
|
|
2791
|
+
Accept: stream ? "text/event-stream" : "application/json"
|
|
2792
|
+
}
|
|
2793
|
+
});
|
|
2794
|
+
if (stream) {
|
|
2795
|
+
if (!resp.body) {
|
|
2796
|
+
throw new Error("Received empty stream body");
|
|
2797
|
+
}
|
|
2798
|
+
return new BraintrustStream(resp.body);
|
|
2799
|
+
} else {
|
|
2800
|
+
const data = await resp.json();
|
|
2801
|
+
return schema ? schema.parse(data) : data;
|
|
2802
|
+
}
|
|
2803
|
+
}
|
|
2804
|
+
|
|
2805
|
+
// src/wrappers/oai.ts
|
|
2806
|
+
import { SpanTypeAttribute as SpanTypeAttribute2 } from "@braintrust/core";
|
|
2807
|
+
import { mergeDicts as mergeDicts2 } from "@braintrust/core";
|
|
2808
|
+
function wrapOpenAI(openai) {
|
|
2809
|
+
if (openai?.chat?.completions?.create) {
|
|
2810
|
+
return wrapOpenAIv4(openai);
|
|
2811
|
+
} else {
|
|
2812
|
+
console.warn("Unsupported OpenAI library (potentially v3). Not wrapping.");
|
|
2813
|
+
return openai;
|
|
2814
|
+
}
|
|
2815
|
+
}
|
|
2816
|
+
globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI;
|
|
2817
|
+
function wrapOpenAIv4(openai) {
|
|
2818
|
+
let completionProxy = new Proxy(openai.chat.completions, {
|
|
2819
|
+
get(target, name, receiver) {
|
|
2820
|
+
const baseVal = Reflect.get(target, name, receiver);
|
|
2821
|
+
if (name === "create") {
|
|
2822
|
+
return wrapChatCompletion(baseVal.bind(target));
|
|
2823
|
+
}
|
|
2824
|
+
return baseVal;
|
|
2825
|
+
}
|
|
2826
|
+
});
|
|
2827
|
+
let chatProxy = new Proxy(openai.chat, {
|
|
2828
|
+
get(target, name, receiver) {
|
|
2829
|
+
if (name === "completions") {
|
|
2830
|
+
return completionProxy;
|
|
2831
|
+
}
|
|
2832
|
+
return Reflect.get(target, name, receiver);
|
|
2833
|
+
}
|
|
2834
|
+
});
|
|
2835
|
+
let embeddingProxy = new Proxy(openai.embeddings, {
|
|
2836
|
+
get(target, name, receiver) {
|
|
2837
|
+
const baseVal = Reflect.get(target, name, receiver);
|
|
2838
|
+
if (name === "create") {
|
|
2839
|
+
return wrapEmbeddings(baseVal.bind(target));
|
|
2840
|
+
}
|
|
2841
|
+
return baseVal;
|
|
2842
|
+
}
|
|
2843
|
+
});
|
|
2844
|
+
let betaProxy;
|
|
2845
|
+
if (openai.beta?.chat?.completions?.stream) {
|
|
2846
|
+
let betaChatCompletionProxy = new Proxy(openai?.beta?.chat.completions, {
|
|
2847
|
+
get(target, name, receiver) {
|
|
2848
|
+
const baseVal = Reflect.get(target, name, receiver);
|
|
2849
|
+
if (name === "stream") {
|
|
2850
|
+
return wrapBetaChatCompletion(baseVal.bind(target));
|
|
2851
|
+
}
|
|
2852
|
+
return baseVal;
|
|
2853
|
+
}
|
|
2854
|
+
});
|
|
2855
|
+
let betaChatProxy = new Proxy(openai.beta.chat, {
|
|
2856
|
+
get(target, name, receiver) {
|
|
2857
|
+
if (name === "completions") {
|
|
2858
|
+
return betaChatCompletionProxy;
|
|
2859
|
+
}
|
|
2860
|
+
return Reflect.get(target, name, receiver);
|
|
2861
|
+
}
|
|
2862
|
+
});
|
|
2863
|
+
betaProxy = new Proxy(openai.beta, {
|
|
2864
|
+
get(target, name, receiver) {
|
|
2865
|
+
if (name === "chat") {
|
|
2866
|
+
return betaChatProxy;
|
|
2867
|
+
}
|
|
2868
|
+
return Reflect.get(target, name, receiver);
|
|
2869
|
+
}
|
|
2870
|
+
});
|
|
2871
|
+
}
|
|
2872
|
+
let proxy = new Proxy(openai, {
|
|
2873
|
+
get(target, name, receiver) {
|
|
2874
|
+
if (name === "chat") {
|
|
2875
|
+
return chatProxy;
|
|
2876
|
+
}
|
|
2877
|
+
if (name === "embeddings") {
|
|
2878
|
+
return embeddingProxy;
|
|
2879
|
+
}
|
|
2880
|
+
if (name === "beta" && betaProxy) {
|
|
2881
|
+
return betaProxy;
|
|
2882
|
+
}
|
|
2883
|
+
return Reflect.get(target, name, receiver);
|
|
2884
|
+
}
|
|
2885
|
+
});
|
|
2886
|
+
return proxy;
|
|
2887
|
+
}
|
|
2888
|
+
function wrapBetaChatCompletion(completion) {
|
|
2889
|
+
return (allParams) => {
|
|
2890
|
+
const { span_info: _, ...params } = allParams;
|
|
2891
|
+
const span = startSpan(
|
|
2892
|
+
mergeDicts2(
|
|
2893
|
+
{
|
|
2894
|
+
name: "Chat Completion",
|
|
2895
|
+
spanAttributes: {
|
|
2896
|
+
type: SpanTypeAttribute2.LLM
|
|
2897
|
+
}
|
|
2898
|
+
},
|
|
2899
|
+
parseChatCompletionParams(allParams)
|
|
2900
|
+
)
|
|
2901
|
+
);
|
|
2902
|
+
const startTime = getCurrentUnixTimestamp();
|
|
2903
|
+
const ret = completion(params);
|
|
2904
|
+
let first = true;
|
|
2905
|
+
ret.on("chunk", (_chunk) => {
|
|
2906
|
+
if (first) {
|
|
2907
|
+
const now2 = getCurrentUnixTimestamp();
|
|
2908
|
+
span.log({
|
|
2909
|
+
metrics: {
|
|
2910
|
+
time_to_first_token: now2 - startTime
|
|
2911
|
+
}
|
|
2912
|
+
});
|
|
2913
|
+
first = false;
|
|
2914
|
+
}
|
|
2915
|
+
});
|
|
2916
|
+
ret.on("chatCompletion", (completion2) => {
|
|
2917
|
+
span.log({
|
|
2918
|
+
output: completion2.choices
|
|
2919
|
+
});
|
|
2920
|
+
});
|
|
2921
|
+
ret.on("end", () => {
|
|
2922
|
+
span.end();
|
|
2923
|
+
});
|
|
2924
|
+
return ret;
|
|
2925
|
+
};
|
|
2926
|
+
}
|
|
2927
|
+
var LEGACY_CACHED_HEADER = "x-cached";
|
|
2928
|
+
var X_CACHED_HEADER = "x-bt-cached";
|
|
2929
|
+
function parseCachedHeader(value) {
|
|
2930
|
+
return isEmpty(value) ? void 0 : ["true", "hit"].includes(value.toLowerCase()) ? 1 : 0;
|
|
2931
|
+
}
|
|
2932
|
+
function logHeaders(response, span) {
|
|
2933
|
+
const cachedHeader = response.headers.get(X_CACHED_HEADER);
|
|
2934
|
+
if (isEmpty(cachedHeader)) {
|
|
2935
|
+
const legacyCacheHeader = response.headers.get(LEGACY_CACHED_HEADER);
|
|
2936
|
+
if (!isEmpty(legacyCacheHeader)) {
|
|
2937
|
+
span.log({
|
|
2938
|
+
metrics: {
|
|
2939
|
+
cached: parseCachedHeader(legacyCacheHeader)
|
|
2940
|
+
}
|
|
2941
|
+
});
|
|
2942
|
+
}
|
|
2943
|
+
} else {
|
|
2944
|
+
span.log({
|
|
2945
|
+
metrics: {
|
|
2946
|
+
cached: parseCachedHeader(cachedHeader)
|
|
2947
|
+
}
|
|
2948
|
+
});
|
|
2949
|
+
}
|
|
2950
|
+
}
|
|
2951
|
+
function wrapChatCompletion(completion) {
|
|
2952
|
+
return async (allParams, options) => {
|
|
2953
|
+
const { span_info: _, ...params } = allParams;
|
|
2954
|
+
const span = startSpan(
|
|
2955
|
+
mergeDicts2(
|
|
2956
|
+
{
|
|
2957
|
+
name: "Chat Completion",
|
|
2958
|
+
spanAttributes: {
|
|
2959
|
+
type: SpanTypeAttribute2.LLM
|
|
2960
|
+
}
|
|
2961
|
+
},
|
|
2962
|
+
parseChatCompletionParams(allParams)
|
|
2963
|
+
)
|
|
2964
|
+
);
|
|
2965
|
+
const startTime = getCurrentUnixTimestamp();
|
|
2966
|
+
if (params.stream) {
|
|
2967
|
+
const { data: ret, response } = await completion(
|
|
2968
|
+
// We could get rid of this type coercion if we could somehow enforce
|
|
2969
|
+
// that `P extends ChatParams` BUT does not have the property
|
|
2970
|
+
// `span_info`.
|
|
2971
|
+
params,
|
|
2972
|
+
options
|
|
2973
|
+
).withResponse();
|
|
2974
|
+
logHeaders(response, span);
|
|
2975
|
+
const wrapperStream = new WrapperStream(span, startTime, ret.iterator());
|
|
2976
|
+
ret.iterator = () => wrapperStream[Symbol.asyncIterator]();
|
|
2977
|
+
return ret;
|
|
2978
|
+
} else {
|
|
2979
|
+
try {
|
|
2980
|
+
const { data: ret, response } = await completion(
|
|
2981
|
+
params,
|
|
2982
|
+
options
|
|
2983
|
+
).withResponse();
|
|
2984
|
+
logHeaders(response, span);
|
|
2985
|
+
const { messages, ...rest } = params;
|
|
2986
|
+
span.log({
|
|
2987
|
+
input: messages,
|
|
2988
|
+
metadata: {
|
|
2989
|
+
...rest
|
|
2990
|
+
},
|
|
2991
|
+
output: ret.choices,
|
|
2992
|
+
metrics: {
|
|
2993
|
+
time_to_first_token: getCurrentUnixTimestamp() - startTime,
|
|
2994
|
+
tokens: ret.usage?.total_tokens,
|
|
2995
|
+
prompt_tokens: ret.usage?.prompt_tokens,
|
|
2996
|
+
completion_tokens: ret.usage?.completion_tokens
|
|
2997
|
+
}
|
|
2998
|
+
});
|
|
2999
|
+
return ret;
|
|
3000
|
+
} finally {
|
|
3001
|
+
span.end();
|
|
3002
|
+
}
|
|
3003
|
+
}
|
|
3004
|
+
};
|
|
3005
|
+
}
|
|
3006
|
+
function parseChatCompletionParams(allParams) {
|
|
3007
|
+
const { span_info, ...params } = allParams;
|
|
3008
|
+
const { metadata: spanInfoMetadata, ...spanInfoRest } = span_info ?? {};
|
|
3009
|
+
let ret = {
|
|
3010
|
+
...spanInfoRest,
|
|
3011
|
+
event: {
|
|
3012
|
+
metadata: spanInfoMetadata
|
|
3013
|
+
}
|
|
3014
|
+
};
|
|
3015
|
+
const { messages, ...paramsRest } = params;
|
|
3016
|
+
return mergeDicts2(ret, { event: { input: messages, metadata: paramsRest } });
|
|
3017
|
+
}
|
|
3018
|
+
function wrapEmbeddings(create) {
|
|
3019
|
+
return async (allParams, options) => {
|
|
3020
|
+
const { span_info: _, ...params } = allParams;
|
|
3021
|
+
return traced(
|
|
3022
|
+
async (span) => {
|
|
3023
|
+
const { data: result, response } = await create(
|
|
3024
|
+
params,
|
|
3025
|
+
options
|
|
3026
|
+
).withResponse();
|
|
3027
|
+
logHeaders(response, span);
|
|
3028
|
+
const embedding_length = result.data[0].embedding.length;
|
|
3029
|
+
span.log({
|
|
3030
|
+
// TODO: Add a flag to control whether to log the full embedding vector,
|
|
3031
|
+
// possibly w/ JSON compression.
|
|
3032
|
+
output: { embedding_length },
|
|
3033
|
+
metrics: {
|
|
3034
|
+
tokens: result.usage?.total_tokens,
|
|
3035
|
+
prompt_tokens: result.usage?.prompt_tokens
|
|
3036
|
+
}
|
|
3037
|
+
});
|
|
3038
|
+
return result;
|
|
3039
|
+
},
|
|
3040
|
+
mergeDicts2(
|
|
3041
|
+
{
|
|
3042
|
+
name: "Embedding",
|
|
3043
|
+
spanAttributes: {
|
|
3044
|
+
type: SpanTypeAttribute2.LLM
|
|
3045
|
+
}
|
|
3046
|
+
},
|
|
3047
|
+
parseEmbeddingParams(allParams)
|
|
3048
|
+
)
|
|
3049
|
+
);
|
|
3050
|
+
};
|
|
3051
|
+
}
|
|
3052
|
+
function parseEmbeddingParams(allParams) {
|
|
3053
|
+
const { span_info, ...params } = allParams;
|
|
3054
|
+
const { metadata: spanInfoMetadata, ...spanInfoRest } = span_info ?? {};
|
|
3055
|
+
let ret = {
|
|
3056
|
+
...spanInfoRest,
|
|
3057
|
+
event: {
|
|
3058
|
+
metadata: spanInfoMetadata
|
|
3059
|
+
}
|
|
3060
|
+
};
|
|
3061
|
+
const { input, ...paramsRest } = params;
|
|
3062
|
+
return mergeDicts2(ret, { event: { input, metadata: paramsRest } });
|
|
3063
|
+
}
|
|
3064
|
+
function postprocessStreamingResults(allResults) {
|
|
3065
|
+
let role = void 0;
|
|
3066
|
+
let content = void 0;
|
|
3067
|
+
let tool_calls = void 0;
|
|
3068
|
+
let finish_reason = void 0;
|
|
3069
|
+
let metrics = {};
|
|
3070
|
+
for (const result of allResults) {
|
|
3071
|
+
if (result.usage) {
|
|
3072
|
+
metrics = {
|
|
3073
|
+
...metrics,
|
|
3074
|
+
tokens: result.usage.total_tokens,
|
|
3075
|
+
prompt_tokens: result.usage.prompt_tokens,
|
|
3076
|
+
completion_tokens: result.usage.completion_tokens
|
|
3077
|
+
};
|
|
3078
|
+
}
|
|
3079
|
+
const delta = result.choices?.[0]?.delta;
|
|
3080
|
+
if (!delta) {
|
|
3081
|
+
continue;
|
|
3082
|
+
}
|
|
3083
|
+
if (!role && delta.role) {
|
|
3084
|
+
role = delta.role;
|
|
3085
|
+
}
|
|
3086
|
+
if (delta.finish_reason) {
|
|
3087
|
+
finish_reason = delta.finish_reason;
|
|
3088
|
+
}
|
|
3089
|
+
if (delta.content) {
|
|
3090
|
+
content = (content || "") + delta.content;
|
|
3091
|
+
}
|
|
3092
|
+
if (delta.tool_calls) {
|
|
3093
|
+
if (!tool_calls) {
|
|
3094
|
+
tool_calls = [
|
|
3095
|
+
{
|
|
3096
|
+
id: delta.tool_calls[0].id,
|
|
3097
|
+
type: delta.tool_calls[0].type,
|
|
3098
|
+
function: delta.tool_calls[0].function
|
|
3099
|
+
}
|
|
3100
|
+
];
|
|
3101
|
+
} else {
|
|
3102
|
+
tool_calls[0].function.arguments += delta.tool_calls[0].function.arguments;
|
|
3103
|
+
}
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
return {
|
|
3107
|
+
metrics,
|
|
3108
|
+
output: [
|
|
3109
|
+
{
|
|
3110
|
+
index: 0,
|
|
3111
|
+
message: {
|
|
3112
|
+
role,
|
|
3113
|
+
content,
|
|
3114
|
+
tool_calls
|
|
3115
|
+
},
|
|
3116
|
+
logprobs: null,
|
|
3117
|
+
finish_reason
|
|
3118
|
+
}
|
|
3119
|
+
]
|
|
3120
|
+
};
|
|
3121
|
+
}
|
|
3122
|
+
var WrapperStream = class {
|
|
3123
|
+
span;
|
|
3124
|
+
iter;
|
|
3125
|
+
startTime;
|
|
3126
|
+
constructor(span, startTime, iter) {
|
|
3127
|
+
this.span = span;
|
|
3128
|
+
this.iter = iter;
|
|
3129
|
+
this.startTime = startTime;
|
|
3130
|
+
}
|
|
3131
|
+
async *[Symbol.asyncIterator]() {
|
|
3132
|
+
let first = true;
|
|
3133
|
+
let allResults = [];
|
|
3134
|
+
try {
|
|
3135
|
+
for await (const item of this.iter) {
|
|
3136
|
+
if (first) {
|
|
3137
|
+
const now2 = getCurrentUnixTimestamp();
|
|
3138
|
+
this.span.log({
|
|
3139
|
+
metrics: {
|
|
3140
|
+
time_to_first_token: now2 - this.startTime
|
|
3141
|
+
}
|
|
3142
|
+
});
|
|
3143
|
+
first = false;
|
|
3144
|
+
}
|
|
3145
|
+
allResults.push(item);
|
|
3146
|
+
yield item;
|
|
3147
|
+
}
|
|
3148
|
+
this.span.log({
|
|
3149
|
+
...postprocessStreamingResults(allResults)
|
|
3150
|
+
});
|
|
3151
|
+
} finally {
|
|
3152
|
+
this.span.end();
|
|
3153
|
+
}
|
|
3154
|
+
}
|
|
3155
|
+
};
|
|
3156
|
+
|
|
3157
|
+
// src/browser.ts
|
|
3158
|
+
configureBrowser();
|
|
3159
|
+
export {
|
|
3160
|
+
BraintrustState,
|
|
3161
|
+
BraintrustStream,
|
|
3162
|
+
Dataset,
|
|
3163
|
+
Experiment,
|
|
3164
|
+
LEGACY_CACHED_HEADER,
|
|
3165
|
+
Logger,
|
|
3166
|
+
NOOP_SPAN,
|
|
3167
|
+
NoopSpan,
|
|
3168
|
+
Prompt,
|
|
3169
|
+
ReadonlyExperiment,
|
|
3170
|
+
SpanImpl,
|
|
3171
|
+
X_CACHED_HEADER,
|
|
3172
|
+
_internalGetGlobalState,
|
|
3173
|
+
_internalSetInitialState,
|
|
3174
|
+
createFinalValuePassThroughStream,
|
|
3175
|
+
currentExperiment,
|
|
3176
|
+
currentLogger,
|
|
3177
|
+
currentSpan,
|
|
3178
|
+
devNullWritableStream,
|
|
3179
|
+
flush,
|
|
3180
|
+
getSpanParentObject,
|
|
3181
|
+
init,
|
|
3182
|
+
initDataset,
|
|
3183
|
+
initExperiment,
|
|
3184
|
+
initLogger,
|
|
3185
|
+
invoke,
|
|
3186
|
+
loadPrompt,
|
|
3187
|
+
log,
|
|
3188
|
+
login,
|
|
3189
|
+
loginToState,
|
|
3190
|
+
newId,
|
|
3191
|
+
parseCachedHeader,
|
|
3192
|
+
setFetch,
|
|
3193
|
+
startSpan,
|
|
3194
|
+
summarize,
|
|
3195
|
+
traceable,
|
|
3196
|
+
traced,
|
|
3197
|
+
withDataset,
|
|
3198
|
+
withExperiment,
|
|
3199
|
+
withLogger,
|
|
3200
|
+
wrapOpenAI,
|
|
3201
|
+
wrapOpenAIv4,
|
|
3202
|
+
wrapTraced
|
|
3203
|
+
};
|