braintrust 0.0.166 → 0.0.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.mts +184 -41
- package/dist/browser.d.ts +184 -41
- package/dist/browser.js +348 -101
- package/dist/browser.mjs +349 -103
- package/dist/cli.js +397 -85
- package/dist/index.d.mts +186 -44
- package/dist/index.d.ts +186 -44
- package/dist/index.js +369 -105
- package/dist/index.mjs +370 -107
- package/package.json +7 -2
- package/typedoc.json +1 -0
package/dist/cli.js
CHANGED
|
@@ -1236,7 +1236,7 @@ var require_package = __commonJS({
|
|
|
1236
1236
|
"package.json"(exports2, module2) {
|
|
1237
1237
|
module2.exports = {
|
|
1238
1238
|
name: "braintrust",
|
|
1239
|
-
version: "0.0.
|
|
1239
|
+
version: "0.0.168",
|
|
1240
1240
|
description: "SDK for integrating Braintrust",
|
|
1241
1241
|
repository: {
|
|
1242
1242
|
type: "git",
|
|
@@ -1264,6 +1264,11 @@ var require_package = __commonJS({
|
|
|
1264
1264
|
module: "./dist/index.mjs",
|
|
1265
1265
|
require: "./dist/index.js"
|
|
1266
1266
|
},
|
|
1267
|
+
"./browser": {
|
|
1268
|
+
import: "./dist/browser.mjs",
|
|
1269
|
+
module: "./dist/browser.mjs",
|
|
1270
|
+
require: "./dist/browser.js"
|
|
1271
|
+
},
|
|
1267
1272
|
"./ai-sdk": {
|
|
1268
1273
|
types: "./ai-sdk/dist/index.d.ts",
|
|
1269
1274
|
import: "./ai-sdk/dist/index.mjs",
|
|
@@ -1306,7 +1311,7 @@ var require_package = __commonJS({
|
|
|
1306
1311
|
},
|
|
1307
1312
|
dependencies: {
|
|
1308
1313
|
"@ai-sdk/provider": "^0.0.11",
|
|
1309
|
-
"@braintrust/core": "0.0.
|
|
1314
|
+
"@braintrust/core": "0.0.64",
|
|
1310
1315
|
"@next/env": "^14.2.3",
|
|
1311
1316
|
"@vercel/functions": "^1.0.2",
|
|
1312
1317
|
ai: "^3.2.16",
|
|
@@ -1714,6 +1719,9 @@ var NoopSpan = class {
|
|
|
1714
1719
|
async export() {
|
|
1715
1720
|
return "";
|
|
1716
1721
|
}
|
|
1722
|
+
async permalink() {
|
|
1723
|
+
return "";
|
|
1724
|
+
}
|
|
1717
1725
|
async flush() {
|
|
1718
1726
|
}
|
|
1719
1727
|
close(args) {
|
|
@@ -1992,12 +2000,14 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
1992
2000
|
([k, v]) => v !== void 0 ? typeof v === "string" ? [[k, v]] : v.map((x) => [k, x]) : []
|
|
1993
2001
|
) : []
|
|
1994
2002
|
).toString();
|
|
2003
|
+
const this_fetch = this.fetch;
|
|
2004
|
+
const this_headers = this.headers;
|
|
1995
2005
|
return await checkResponse(
|
|
1996
2006
|
// Using toString() here makes it work with isomorphic fetch
|
|
1997
|
-
await
|
|
2007
|
+
await this_fetch(url.toString(), {
|
|
1998
2008
|
headers: {
|
|
1999
2009
|
Accept: "application/json",
|
|
2000
|
-
...
|
|
2010
|
+
...this_headers,
|
|
2001
2011
|
...headers
|
|
2002
2012
|
},
|
|
2003
2013
|
keepalive: true,
|
|
@@ -2007,13 +2017,16 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
2007
2017
|
}
|
|
2008
2018
|
async post(path9, params, config3) {
|
|
2009
2019
|
const { headers, ...rest } = config3 || {};
|
|
2020
|
+
const this_fetch = this.fetch;
|
|
2021
|
+
const this_base_url = this.base_url;
|
|
2022
|
+
const this_headers = this.headers;
|
|
2010
2023
|
return await checkResponse(
|
|
2011
|
-
await
|
|
2024
|
+
await this_fetch((0, import_core._urljoin)(this_base_url, path9), {
|
|
2012
2025
|
method: "POST",
|
|
2013
2026
|
headers: {
|
|
2014
2027
|
Accept: "application/json",
|
|
2015
2028
|
"Content-Type": "application/json",
|
|
2016
|
-
...
|
|
2029
|
+
...this_headers,
|
|
2017
2030
|
...headers
|
|
2018
2031
|
},
|
|
2019
2032
|
body: typeof params === "string" ? params : params ? JSON.stringify(params) : void 0,
|
|
@@ -2046,6 +2059,167 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
2046
2059
|
return await resp.json();
|
|
2047
2060
|
}
|
|
2048
2061
|
};
|
|
2062
|
+
var Attachment = class {
|
|
2063
|
+
/**
|
|
2064
|
+
* The object that replaces this `Attachment` at upload time.
|
|
2065
|
+
*/
|
|
2066
|
+
reference;
|
|
2067
|
+
uploader;
|
|
2068
|
+
data;
|
|
2069
|
+
state;
|
|
2070
|
+
// For debug logging only.
|
|
2071
|
+
dataDebugString;
|
|
2072
|
+
/**
|
|
2073
|
+
* Construct an attachment.
|
|
2074
|
+
*
|
|
2075
|
+
* @param data A string representing the path of the file on disk, or a
|
|
2076
|
+
* `Blob`/`ArrayBuffer` with the file's contents. The caller is responsible
|
|
2077
|
+
* for ensuring the file/blob/buffer is not modified until upload is complete.
|
|
2078
|
+
*
|
|
2079
|
+
* @param filename The desired name of the file in Braintrust after uploading.
|
|
2080
|
+
* This parameter is for visualization purposes only and has no effect on
|
|
2081
|
+
* attachment storage.
|
|
2082
|
+
*
|
|
2083
|
+
* @param contentType The MIME type of the file.
|
|
2084
|
+
*
|
|
2085
|
+
* @param state (Optional) For internal use.
|
|
2086
|
+
*/
|
|
2087
|
+
constructor({ data, filename, contentType, state }) {
|
|
2088
|
+
this.reference = {
|
|
2089
|
+
type: import_typespecs2.BRAINTRUST_ATTACHMENT,
|
|
2090
|
+
filename,
|
|
2091
|
+
content_type: contentType,
|
|
2092
|
+
key: newId()
|
|
2093
|
+
};
|
|
2094
|
+
this.state = state;
|
|
2095
|
+
this.dataDebugString = typeof data === "string" ? data : "<in-memory data>";
|
|
2096
|
+
this.data = this.initData(data);
|
|
2097
|
+
this.uploader = this.initUploader();
|
|
2098
|
+
}
|
|
2099
|
+
/**
|
|
2100
|
+
* On first access, (1) reads the attachment from disk if needed, (2)
|
|
2101
|
+
* authenticates with the data plane to request a signed URL, (3) uploads to
|
|
2102
|
+
* object store, and (4) updates the attachment.
|
|
2103
|
+
*
|
|
2104
|
+
* @returns The attachment status.
|
|
2105
|
+
*/
|
|
2106
|
+
async upload() {
|
|
2107
|
+
return await this.uploader.get();
|
|
2108
|
+
}
|
|
2109
|
+
/**
|
|
2110
|
+
* A human-readable description for logging and debugging.
|
|
2111
|
+
*
|
|
2112
|
+
* @returns The debug object. The return type is not stable and may change in
|
|
2113
|
+
* a future release.
|
|
2114
|
+
*/
|
|
2115
|
+
debugInfo() {
|
|
2116
|
+
return {
|
|
2117
|
+
inputData: this.dataDebugString,
|
|
2118
|
+
reference: this.reference,
|
|
2119
|
+
state: this.state
|
|
2120
|
+
};
|
|
2121
|
+
}
|
|
2122
|
+
initUploader() {
|
|
2123
|
+
const doUpload = async (conn, orgId) => {
|
|
2124
|
+
const requestParams = {
|
|
2125
|
+
key: this.reference.key,
|
|
2126
|
+
filename: this.reference.filename,
|
|
2127
|
+
content_type: this.reference.content_type,
|
|
2128
|
+
org_id: orgId
|
|
2129
|
+
};
|
|
2130
|
+
const [metadataPromiseResult, dataPromiseResult] = await Promise.allSettled([
|
|
2131
|
+
conn.post("/attachment", requestParams),
|
|
2132
|
+
this.data.get()
|
|
2133
|
+
]);
|
|
2134
|
+
if (metadataPromiseResult.status === "rejected") {
|
|
2135
|
+
const errorStr = JSON.stringify(metadataPromiseResult.reason);
|
|
2136
|
+
throw new Error(
|
|
2137
|
+
`Failed to request signed URL from API server: ${errorStr}`
|
|
2138
|
+
);
|
|
2139
|
+
}
|
|
2140
|
+
if (dataPromiseResult.status === "rejected") {
|
|
2141
|
+
const errorStr = JSON.stringify(dataPromiseResult.reason);
|
|
2142
|
+
throw new Error(`Failed to read file: ${errorStr}`);
|
|
2143
|
+
}
|
|
2144
|
+
const metadataResponse = metadataPromiseResult.value;
|
|
2145
|
+
const data = dataPromiseResult.value;
|
|
2146
|
+
let signedUrl;
|
|
2147
|
+
let headers;
|
|
2148
|
+
try {
|
|
2149
|
+
({ signedUrl, headers } = import_zod2.z.object({
|
|
2150
|
+
signedUrl: import_zod2.z.string().url(),
|
|
2151
|
+
headers: import_zod2.z.record(import_zod2.z.string())
|
|
2152
|
+
}).parse(await metadataResponse.json()));
|
|
2153
|
+
} catch (error2) {
|
|
2154
|
+
if (error2 instanceof import_zod2.ZodError) {
|
|
2155
|
+
const errorStr = JSON.stringify(error2.flatten());
|
|
2156
|
+
throw new Error(`Invalid response from API server: ${errorStr}`);
|
|
2157
|
+
}
|
|
2158
|
+
throw error2;
|
|
2159
|
+
}
|
|
2160
|
+
let objectStoreResponse;
|
|
2161
|
+
try {
|
|
2162
|
+
objectStoreResponse = await checkResponse(
|
|
2163
|
+
await fetch(signedUrl, {
|
|
2164
|
+
method: "PUT",
|
|
2165
|
+
headers,
|
|
2166
|
+
body: data
|
|
2167
|
+
})
|
|
2168
|
+
);
|
|
2169
|
+
} catch (error2) {
|
|
2170
|
+
if (error2 instanceof FailedHTTPResponse) {
|
|
2171
|
+
throw new Error(
|
|
2172
|
+
`Failed to upload attachment to object store: ${error2.status} ${error2.text} ${error2.data}`
|
|
2173
|
+
);
|
|
2174
|
+
}
|
|
2175
|
+
throw error2;
|
|
2176
|
+
}
|
|
2177
|
+
return { signedUrl, metadataResponse, objectStoreResponse };
|
|
2178
|
+
};
|
|
2179
|
+
const errorWrapper = async () => {
|
|
2180
|
+
const status = { upload_status: "done" };
|
|
2181
|
+
const state = this.state ?? _globalState;
|
|
2182
|
+
await state.login({});
|
|
2183
|
+
const conn = state.apiConn();
|
|
2184
|
+
const orgId = state.orgId ?? "";
|
|
2185
|
+
try {
|
|
2186
|
+
await doUpload(conn, orgId);
|
|
2187
|
+
} catch (error2) {
|
|
2188
|
+
status.upload_status = "error";
|
|
2189
|
+
status.error_message = error2 instanceof Error ? error2.message : JSON.stringify(error2);
|
|
2190
|
+
}
|
|
2191
|
+
const requestParams = {
|
|
2192
|
+
key: this.reference.key,
|
|
2193
|
+
org_id: orgId,
|
|
2194
|
+
status
|
|
2195
|
+
};
|
|
2196
|
+
const statusResponse = await conn.post(
|
|
2197
|
+
"/attachment/status",
|
|
2198
|
+
requestParams
|
|
2199
|
+
);
|
|
2200
|
+
if (!statusResponse.ok) {
|
|
2201
|
+
const errorStr = JSON.stringify(statusResponse);
|
|
2202
|
+
throw new Error(`Couldn't log attachment status: ${errorStr}`);
|
|
2203
|
+
}
|
|
2204
|
+
return status;
|
|
2205
|
+
};
|
|
2206
|
+
return new LazyValue(errorWrapper);
|
|
2207
|
+
}
|
|
2208
|
+
initData(data) {
|
|
2209
|
+
if (typeof data === "string") {
|
|
2210
|
+
const readFile3 = isomorph_default.readFile;
|
|
2211
|
+
if (!readFile3) {
|
|
2212
|
+
throw new Error(
|
|
2213
|
+
`This platform does not support reading the filesystem. Construct the Attachment
|
|
2214
|
+
with a Blob/ArrayBuffer, or run the program on Node.js.`
|
|
2215
|
+
);
|
|
2216
|
+
}
|
|
2217
|
+
return new LazyValue(async () => new Blob([await readFile3(data)]));
|
|
2218
|
+
} else {
|
|
2219
|
+
return new LazyValue(async () => new Blob([data]));
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
};
|
|
2049
2223
|
function logFeedbackImpl(state, parentObjectType, parentObjectId, {
|
|
2050
2224
|
id,
|
|
2051
2225
|
expected,
|
|
@@ -2070,7 +2244,7 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
|
|
|
2070
2244
|
expected,
|
|
2071
2245
|
tags
|
|
2072
2246
|
});
|
|
2073
|
-
let { metadata, ...updateEvent } = validatedEvent;
|
|
2247
|
+
let { metadata, ...updateEvent } = deepCopyEvent(validatedEvent);
|
|
2074
2248
|
updateEvent = Object.fromEntries(
|
|
2075
2249
|
Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
|
|
2076
2250
|
);
|
|
@@ -2119,10 +2293,12 @@ function updateSpanImpl({
|
|
|
2119
2293
|
id,
|
|
2120
2294
|
event
|
|
2121
2295
|
}) {
|
|
2122
|
-
const updateEvent =
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2296
|
+
const updateEvent = deepCopyEvent(
|
|
2297
|
+
validateAndSanitizeExperimentLogPartialArgs({
|
|
2298
|
+
id,
|
|
2299
|
+
...event
|
|
2300
|
+
})
|
|
2301
|
+
);
|
|
2126
2302
|
const parentIds = async () => new import_core.SpanComponentsV3({
|
|
2127
2303
|
object_type: parentObjectType,
|
|
2128
2304
|
object_id: await parentObjectId.get()
|
|
@@ -2159,6 +2335,53 @@ function spanComponentsToObjectIdLambda(state, components) {
|
|
|
2159
2335
|
throw new Error(`Unknown object type: ${x}`);
|
|
2160
2336
|
}
|
|
2161
2337
|
}
|
|
2338
|
+
async function spanComponentsToObjectId({
|
|
2339
|
+
components,
|
|
2340
|
+
state
|
|
2341
|
+
}) {
|
|
2342
|
+
return await spanComponentsToObjectIdLambda(
|
|
2343
|
+
state ?? _globalState,
|
|
2344
|
+
components
|
|
2345
|
+
)();
|
|
2346
|
+
}
|
|
2347
|
+
async function permalink(slug, opts) {
|
|
2348
|
+
const state = opts?.state ?? _globalState;
|
|
2349
|
+
const getOrgName = async () => {
|
|
2350
|
+
if (opts?.orgName) {
|
|
2351
|
+
return opts.orgName;
|
|
2352
|
+
}
|
|
2353
|
+
await state.login({});
|
|
2354
|
+
if (!state.orgName) {
|
|
2355
|
+
throw new Error(
|
|
2356
|
+
"Must either provide orgName explicitly or be logged in to a specific org"
|
|
2357
|
+
);
|
|
2358
|
+
}
|
|
2359
|
+
return state.orgName;
|
|
2360
|
+
};
|
|
2361
|
+
const getAppUrl = async () => {
|
|
2362
|
+
if (opts?.appUrl) {
|
|
2363
|
+
return opts.appUrl;
|
|
2364
|
+
}
|
|
2365
|
+
await state.login({});
|
|
2366
|
+
if (!state.appUrl) {
|
|
2367
|
+
throw new Error("Must either provide appUrl explicitly or be logged in");
|
|
2368
|
+
}
|
|
2369
|
+
return state.appUrl;
|
|
2370
|
+
};
|
|
2371
|
+
const components = import_core.SpanComponentsV3.fromStr(slug);
|
|
2372
|
+
const object_type = (0, import_core.spanObjectTypeV3ToString)(components.data.object_type);
|
|
2373
|
+
const [orgName, appUrl, object_id] = await Promise.all([
|
|
2374
|
+
getOrgName(),
|
|
2375
|
+
getAppUrl(),
|
|
2376
|
+
spanComponentsToObjectId({ components, state })
|
|
2377
|
+
]);
|
|
2378
|
+
const id = components.data.row_id;
|
|
2379
|
+
if (!id) {
|
|
2380
|
+
throw new Error("Span slug does not refer to an individual row");
|
|
2381
|
+
}
|
|
2382
|
+
const urlParams = new URLSearchParams({ object_type, object_id, id });
|
|
2383
|
+
return `${appUrl}/app/${orgName}/object?${urlParams}`;
|
|
2384
|
+
}
|
|
2162
2385
|
function startSpanParentArgs(args) {
|
|
2163
2386
|
let argParentObjectId = void 0;
|
|
2164
2387
|
let argParentSpanIds = void 0;
|
|
@@ -2256,7 +2479,7 @@ var Logger = class {
|
|
|
2256
2479
|
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
2257
2480
|
* @param options Additional logging options
|
|
2258
2481
|
* @param options.allowConcurrentWithSpans in rare cases where you need to log at the top level separately from spans on the logger elsewhere, set this to true.
|
|
2259
|
-
*
|
|
2482
|
+
* @returns The `id` of the logged event.
|
|
2260
2483
|
*/
|
|
2261
2484
|
log(event, options) {
|
|
2262
2485
|
if (this.calledStartSpan && !options?.allowConcurrentWithSpans) {
|
|
@@ -2279,7 +2502,7 @@ var Logger = class {
|
|
|
2279
2502
|
/**
|
|
2280
2503
|
* Create a new toplevel span underneath the logger. The name defaults to "root".
|
|
2281
2504
|
*
|
|
2282
|
-
* See
|
|
2505
|
+
* See {@link Span.traced} for full details.
|
|
2283
2506
|
*/
|
|
2284
2507
|
traced(callback, args) {
|
|
2285
2508
|
const { setCurrent, ...argsRest } = args ?? {};
|
|
@@ -2313,7 +2536,7 @@ var Logger = class {
|
|
|
2313
2536
|
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
2314
2537
|
* so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
|
|
2315
2538
|
*
|
|
2316
|
-
* See
|
|
2539
|
+
* See {@link traced} for full details.
|
|
2317
2540
|
*/
|
|
2318
2541
|
startSpan(args) {
|
|
2319
2542
|
this.calledStartSpan = true;
|
|
@@ -2353,7 +2576,7 @@ var Logger = class {
|
|
|
2353
2576
|
* Update a span in the experiment using its id. It is important that you only update a span once the original span has been fully written and flushed,
|
|
2354
2577
|
* since otherwise updates to the span may conflict with the original span.
|
|
2355
2578
|
*
|
|
2356
|
-
* @param event The event data to update the span with. Must include `id`. See
|
|
2579
|
+
* @param event The event data to update the span with. Must include `id`. See {@link Experiment.log} for a full list of valid fields.
|
|
2357
2580
|
*/
|
|
2358
2581
|
updateSpan(event) {
|
|
2359
2582
|
const { id, ...eventRest } = event;
|
|
@@ -2369,7 +2592,9 @@ var Logger = class {
|
|
|
2369
2592
|
});
|
|
2370
2593
|
}
|
|
2371
2594
|
/**
|
|
2372
|
-
* Return a serialized representation of the logger that can be used to start subspans in other places.
|
|
2595
|
+
* Return a serialized representation of the logger that can be used to start subspans in other places.
|
|
2596
|
+
*
|
|
2597
|
+
* See {@link Span.startSpan} for more details.
|
|
2373
2598
|
*/
|
|
2374
2599
|
async export() {
|
|
2375
2600
|
return new import_core.SpanComponentsV3({
|
|
@@ -2409,6 +2634,7 @@ var BackgroundLogger = class _BackgroundLogger {
|
|
|
2409
2634
|
activeFlush = Promise.resolve();
|
|
2410
2635
|
activeFlushResolved = true;
|
|
2411
2636
|
activeFlushError = void 0;
|
|
2637
|
+
onFlushError;
|
|
2412
2638
|
syncFlush = false;
|
|
2413
2639
|
// 6 MB for the AWS lambda gateway (from our own testing).
|
|
2414
2640
|
maxRequestSize = 6 * 1024 * 1024;
|
|
@@ -2472,6 +2698,7 @@ var BackgroundLogger = class _BackgroundLogger {
|
|
|
2472
2698
|
await this.flush();
|
|
2473
2699
|
});
|
|
2474
2700
|
}
|
|
2701
|
+
this.onFlushError = opts.onFlushError;
|
|
2475
2702
|
}
|
|
2476
2703
|
log(items) {
|
|
2477
2704
|
const [addedItems, droppedItems] = (() => {
|
|
@@ -2503,14 +2730,16 @@ var BackgroundLogger = class _BackgroundLogger {
|
|
|
2503
2730
|
if (this.activeFlushError) {
|
|
2504
2731
|
const err = this.activeFlushError;
|
|
2505
2732
|
this.activeFlushError = void 0;
|
|
2506
|
-
|
|
2733
|
+
if (this.syncFlush) {
|
|
2734
|
+
throw err;
|
|
2735
|
+
}
|
|
2507
2736
|
}
|
|
2508
2737
|
}
|
|
2509
2738
|
async flushOnce(args) {
|
|
2510
2739
|
const batchSize = args?.batchSize ?? this.defaultBatchSize;
|
|
2511
2740
|
const wrappedItems = this.items;
|
|
2512
2741
|
this.items = [];
|
|
2513
|
-
const allItems = await this.unwrapLazyValues(wrappedItems);
|
|
2742
|
+
const [allItems, attachments] = await this.unwrapLazyValues(wrappedItems);
|
|
2514
2743
|
if (allItems.length === 0) {
|
|
2515
2744
|
return;
|
|
2516
2745
|
}
|
|
@@ -2542,6 +2771,23 @@ var BackgroundLogger = class _BackgroundLogger {
|
|
|
2542
2771
|
);
|
|
2543
2772
|
}
|
|
2544
2773
|
}
|
|
2774
|
+
const attachmentErrors = [];
|
|
2775
|
+
for (const attachment of attachments) {
|
|
2776
|
+
try {
|
|
2777
|
+
const result = await attachment.upload();
|
|
2778
|
+
if (result.upload_status === "error" && result.error_message) {
|
|
2779
|
+
attachmentErrors.push(new Error(result.error_message));
|
|
2780
|
+
}
|
|
2781
|
+
} catch (error2) {
|
|
2782
|
+
attachmentErrors.push(error2);
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2785
|
+
if (attachmentErrors.length > 0) {
|
|
2786
|
+
throw new AggregateError(
|
|
2787
|
+
attachmentErrors,
|
|
2788
|
+
`Encountered the following errors while uploading attachments:`
|
|
2789
|
+
);
|
|
2790
|
+
}
|
|
2545
2791
|
if (this.items.length > 0) {
|
|
2546
2792
|
await this.flushOnce(args);
|
|
2547
2793
|
}
|
|
@@ -2549,8 +2795,10 @@ var BackgroundLogger = class _BackgroundLogger {
|
|
|
2549
2795
|
async unwrapLazyValues(wrappedItems) {
|
|
2550
2796
|
for (let i = 0; i < this.numTries; ++i) {
|
|
2551
2797
|
try {
|
|
2552
|
-
const
|
|
2553
|
-
|
|
2798
|
+
const items = await Promise.all(wrappedItems.map((x) => x.get()));
|
|
2799
|
+
const attachments = [];
|
|
2800
|
+
items.forEach((item) => extractAttachments(item, attachments));
|
|
2801
|
+
return [(0, import_core.mergeRowBatch)(items), attachments];
|
|
2554
2802
|
} catch (e) {
|
|
2555
2803
|
let errmsg = "Encountered error when constructing records to flush";
|
|
2556
2804
|
const isRetrying = i + 1 < this.numTries;
|
|
@@ -2558,7 +2806,10 @@ var BackgroundLogger = class _BackgroundLogger {
|
|
|
2558
2806
|
errmsg += ". Retrying";
|
|
2559
2807
|
}
|
|
2560
2808
|
console.warn(errmsg);
|
|
2561
|
-
if (!isRetrying
|
|
2809
|
+
if (!isRetrying) {
|
|
2810
|
+
console.warn(
|
|
2811
|
+
`Failed to construct log records to flush after ${this.numTries} attempts. Dropping batch`
|
|
2812
|
+
);
|
|
2562
2813
|
throw e;
|
|
2563
2814
|
} else {
|
|
2564
2815
|
console.warn(e);
|
|
@@ -2566,10 +2817,7 @@ var BackgroundLogger = class _BackgroundLogger {
|
|
|
2566
2817
|
}
|
|
2567
2818
|
}
|
|
2568
2819
|
}
|
|
2569
|
-
|
|
2570
|
-
`Failed to construct log records to flush after ${this.numTries} attempts. Dropping batch`
|
|
2571
|
-
);
|
|
2572
|
-
return [];
|
|
2820
|
+
throw new Error("Impossible");
|
|
2573
2821
|
}
|
|
2574
2822
|
async submitLogsRequest(items) {
|
|
2575
2823
|
const conn = await this.apiConn.get();
|
|
@@ -2585,16 +2833,14 @@ var BackgroundLogger = class _BackgroundLogger {
|
|
|
2585
2833
|
let error2 = void 0;
|
|
2586
2834
|
try {
|
|
2587
2835
|
await conn.post_json("logs3", dataStr);
|
|
2588
|
-
} catch
|
|
2836
|
+
} catch {
|
|
2589
2837
|
try {
|
|
2590
2838
|
const legacyDataS = (0, import_core.constructJsonArray)(
|
|
2591
|
-
items.map(
|
|
2592
|
-
(r) => JSON.stringify((0, import_core.makeLegacyEvent)(JSON.parse(r)))
|
|
2593
|
-
)
|
|
2839
|
+
items.map((r) => JSON.stringify((0, import_core.makeLegacyEvent)(JSON.parse(r))))
|
|
2594
2840
|
);
|
|
2595
2841
|
await conn.post_json("logs", legacyDataS);
|
|
2596
|
-
} catch (
|
|
2597
|
-
error2 =
|
|
2842
|
+
} catch (e) {
|
|
2843
|
+
error2 = e;
|
|
2598
2844
|
}
|
|
2599
2845
|
}
|
|
2600
2846
|
if (error2 === void 0) {
|
|
@@ -2618,7 +2864,10 @@ Error: ${errorText}`;
|
|
|
2618
2864
|
});
|
|
2619
2865
|
this.logFailedPayloadsDir();
|
|
2620
2866
|
}
|
|
2621
|
-
if (!isRetrying
|
|
2867
|
+
if (!isRetrying) {
|
|
2868
|
+
console.warn(
|
|
2869
|
+
`log request failed after ${this.numTries} retries. Dropping batch`
|
|
2870
|
+
);
|
|
2622
2871
|
throw new Error(errMsg);
|
|
2623
2872
|
} else {
|
|
2624
2873
|
console.warn(errMsg);
|
|
@@ -2627,10 +2876,6 @@ Error: ${errorText}`;
|
|
|
2627
2876
|
}
|
|
2628
2877
|
}
|
|
2629
2878
|
}
|
|
2630
|
-
console.warn(
|
|
2631
|
-
`log request failed after ${this.numTries} retries. Dropping batch`
|
|
2632
|
-
);
|
|
2633
|
-
return;
|
|
2634
2879
|
}
|
|
2635
2880
|
registerDroppedItemCount(numItems) {
|
|
2636
2881
|
if (numItems <= 0) {
|
|
@@ -2658,15 +2903,17 @@ Error: ${errorText}`;
|
|
|
2658
2903
|
return;
|
|
2659
2904
|
}
|
|
2660
2905
|
try {
|
|
2661
|
-
const allItems = await this.unwrapLazyValues(wrappedItems);
|
|
2906
|
+
const [allItems, allAttachments] = await this.unwrapLazyValues(wrappedItems);
|
|
2662
2907
|
const dataStr = constructLogs3Data(
|
|
2663
2908
|
allItems.map((x) => JSON.stringify(x))
|
|
2664
2909
|
);
|
|
2910
|
+
const attachmentStr = JSON.stringify(
|
|
2911
|
+
allAttachments.map((a) => a.debugInfo())
|
|
2912
|
+
);
|
|
2913
|
+
const payload = `{"data": ${dataStr}, "attachments": ${attachmentStr}}
|
|
2914
|
+
`;
|
|
2665
2915
|
for (const payloadDir of publishPayloadsDir) {
|
|
2666
|
-
await _BackgroundLogger.writePayloadToDir({
|
|
2667
|
-
payloadDir,
|
|
2668
|
-
payload: dataStr
|
|
2669
|
-
});
|
|
2916
|
+
await _BackgroundLogger.writePayloadToDir({ payloadDir, payload });
|
|
2670
2917
|
}
|
|
2671
2918
|
} catch (e) {
|
|
2672
2919
|
console.error(e);
|
|
@@ -2705,6 +2952,13 @@ Error: ${errorText}`;
|
|
|
2705
2952
|
try {
|
|
2706
2953
|
await this.flushOnce();
|
|
2707
2954
|
} catch (err) {
|
|
2955
|
+
if (err instanceof AggregateError) {
|
|
2956
|
+
for (const e of err.errors) {
|
|
2957
|
+
this.onFlushError?.(e);
|
|
2958
|
+
}
|
|
2959
|
+
} else {
|
|
2960
|
+
this.onFlushError?.(err);
|
|
2961
|
+
}
|
|
2708
2962
|
this.activeFlushError = err;
|
|
2709
2963
|
} finally {
|
|
2710
2964
|
this.activeFlushResolved = true;
|
|
@@ -3163,6 +3417,47 @@ function validateAndSanitizeExperimentLogPartialArgs(event) {
|
|
|
3163
3417
|
return { ...event };
|
|
3164
3418
|
}
|
|
3165
3419
|
}
|
|
3420
|
+
function deepCopyEvent(event) {
|
|
3421
|
+
const attachments = [];
|
|
3422
|
+
const IDENTIFIER = "_bt_internal_saved_attachment";
|
|
3423
|
+
const savedAttachmentSchema = import_zod2.z.strictObject({ [IDENTIFIER]: import_zod2.z.number() });
|
|
3424
|
+
const serialized = JSON.stringify(event, (_k, v) => {
|
|
3425
|
+
if (v instanceof SpanImpl || v instanceof NoopSpan) {
|
|
3426
|
+
return `<span>`;
|
|
3427
|
+
} else if (v instanceof Experiment) {
|
|
3428
|
+
return `<experiment>`;
|
|
3429
|
+
} else if (v instanceof Dataset) {
|
|
3430
|
+
return `<dataset>`;
|
|
3431
|
+
} else if (v instanceof Logger) {
|
|
3432
|
+
return `<logger>`;
|
|
3433
|
+
} else if (v instanceof Attachment) {
|
|
3434
|
+
const idx = attachments.push(v);
|
|
3435
|
+
return { [IDENTIFIER]: idx - 1 };
|
|
3436
|
+
}
|
|
3437
|
+
return v;
|
|
3438
|
+
});
|
|
3439
|
+
const x = JSON.parse(serialized, (_k, v) => {
|
|
3440
|
+
const parsedAttachment = savedAttachmentSchema.safeParse(v);
|
|
3441
|
+
if (parsedAttachment.success) {
|
|
3442
|
+
return attachments[parsedAttachment.data[IDENTIFIER]];
|
|
3443
|
+
}
|
|
3444
|
+
return v;
|
|
3445
|
+
});
|
|
3446
|
+
return x;
|
|
3447
|
+
}
|
|
3448
|
+
function extractAttachments(event, attachments) {
|
|
3449
|
+
for (const [key, value] of Object.entries(event)) {
|
|
3450
|
+
if (value instanceof Attachment) {
|
|
3451
|
+
attachments.push(value);
|
|
3452
|
+
event[key] = value.reference;
|
|
3453
|
+
continue;
|
|
3454
|
+
}
|
|
3455
|
+
if (!(value instanceof Object)) {
|
|
3456
|
+
continue;
|
|
3457
|
+
}
|
|
3458
|
+
extractAttachments(value, attachments);
|
|
3459
|
+
}
|
|
3460
|
+
}
|
|
3166
3461
|
function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
3167
3462
|
if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
|
|
3168
3463
|
throw new Error(
|
|
@@ -3293,10 +3588,9 @@ var Experiment = class extends ObjectFetcher {
|
|
|
3293
3588
|
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
|
|
3294
3589
|
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
3295
3590
|
* @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
|
|
3296
|
-
* @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
|
|
3297
3591
|
* @param options Additional logging options
|
|
3298
3592
|
* @param options.allowConcurrentWithSpans in rare cases where you need to log at the top level separately from spans on the experiment elsewhere, set this to true.
|
|
3299
|
-
*
|
|
3593
|
+
* @returns The `id` of the logged event.
|
|
3300
3594
|
*/
|
|
3301
3595
|
log(event, options) {
|
|
3302
3596
|
if (this.calledStartSpan && !options?.allowConcurrentWithSpans) {
|
|
@@ -3312,7 +3606,7 @@ var Experiment = class extends ObjectFetcher {
|
|
|
3312
3606
|
/**
|
|
3313
3607
|
* Create a new toplevel span underneath the experiment. The name defaults to "root".
|
|
3314
3608
|
*
|
|
3315
|
-
* See
|
|
3609
|
+
* See {@link Span.traced} for full details.
|
|
3316
3610
|
*/
|
|
3317
3611
|
traced(callback, args) {
|
|
3318
3612
|
const { setCurrent, ...argsRest } = args ?? {};
|
|
@@ -3338,7 +3632,7 @@ var Experiment = class extends ObjectFetcher {
|
|
|
3338
3632
|
* where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
|
|
3339
3633
|
* so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
|
|
3340
3634
|
*
|
|
3341
|
-
* See
|
|
3635
|
+
* See {@link traced} for full details.
|
|
3342
3636
|
*/
|
|
3343
3637
|
startSpan(args) {
|
|
3344
3638
|
this.calledStartSpan = true;
|
|
@@ -3450,7 +3744,7 @@ var Experiment = class extends ObjectFetcher {
|
|
|
3450
3744
|
* Update a span in the experiment using its id. It is important that you only update a span once the original span has been fully written and flushed,
|
|
3451
3745
|
* since otherwise updates to the span may conflict with the original span.
|
|
3452
3746
|
*
|
|
3453
|
-
* @param event The event data to update the span with. Must include `id`. See
|
|
3747
|
+
* @param event The event data to update the span with. Must include `id`. See {@link Experiment.log} for a full list of valid fields.
|
|
3454
3748
|
*/
|
|
3455
3749
|
updateSpan(event) {
|
|
3456
3750
|
const { id, ...eventRest } = event;
|
|
@@ -3466,7 +3760,9 @@ var Experiment = class extends ObjectFetcher {
|
|
|
3466
3760
|
});
|
|
3467
3761
|
}
|
|
3468
3762
|
/**
|
|
3469
|
-
* Return a serialized representation of the experiment that can be used to start subspans in other places.
|
|
3763
|
+
* Return a serialized representation of the experiment that can be used to start subspans in other places.
|
|
3764
|
+
*
|
|
3765
|
+
* See {@link Span.startSpan} for more details.
|
|
3470
3766
|
*/
|
|
3471
3767
|
async export() {
|
|
3472
3768
|
return new import_core.SpanComponentsV3({
|
|
@@ -3481,7 +3777,7 @@ var Experiment = class extends ObjectFetcher {
|
|
|
3481
3777
|
return await this.state.bgLogger().flush();
|
|
3482
3778
|
}
|
|
3483
3779
|
/**
|
|
3484
|
-
* This function is deprecated. You can simply remove it from your code.
|
|
3780
|
+
* @deprecated This function is deprecated. You can simply remove it from your code.
|
|
3485
3781
|
*/
|
|
3486
3782
|
async close() {
|
|
3487
3783
|
console.warn(
|
|
@@ -3534,6 +3830,9 @@ var ReadonlyExperiment = class extends ObjectFetcher {
|
|
|
3534
3830
|
}
|
|
3535
3831
|
};
|
|
3536
3832
|
var executionCounter = 0;
|
|
3833
|
+
function newId() {
|
|
3834
|
+
return (0, import_uuid.v4)();
|
|
3835
|
+
}
|
|
3537
3836
|
var SpanImpl = class _SpanImpl {
|
|
3538
3837
|
state;
|
|
3539
3838
|
isMerge;
|
|
@@ -3620,27 +3919,14 @@ var SpanImpl = class _SpanImpl {
|
|
|
3620
3919
|
event,
|
|
3621
3920
|
internalData
|
|
3622
3921
|
});
|
|
3623
|
-
|
|
3922
|
+
const partialRecord = deepCopyEvent({
|
|
3624
3923
|
id: this.id,
|
|
3625
3924
|
span_id: this.spanId,
|
|
3626
3925
|
root_span_id: this.rootSpanId,
|
|
3627
3926
|
span_parents: this.spanParents,
|
|
3628
3927
|
...serializableInternalData,
|
|
3629
3928
|
[import_core.IS_MERGE_FIELD]: this.isMerge
|
|
3630
|
-
};
|
|
3631
|
-
const serializedPartialRecord = JSON.stringify(partialRecord, (_k, v) => {
|
|
3632
|
-
if (v instanceof _SpanImpl) {
|
|
3633
|
-
return `<span>`;
|
|
3634
|
-
} else if (v instanceof Experiment) {
|
|
3635
|
-
return `<experiment>`;
|
|
3636
|
-
} else if (v instanceof Dataset) {
|
|
3637
|
-
return `<dataset>`;
|
|
3638
|
-
} else if (v instanceof Logger) {
|
|
3639
|
-
return `<logger>`;
|
|
3640
|
-
}
|
|
3641
|
-
return v;
|
|
3642
3929
|
});
|
|
3643
|
-
partialRecord = JSON.parse(serializedPartialRecord);
|
|
3644
3930
|
if (partialRecord.metrics?.end) {
|
|
3645
3931
|
this.loggedEndTime = partialRecord.metrics?.end;
|
|
3646
3932
|
}
|
|
@@ -3726,6 +4012,11 @@ var SpanImpl = class _SpanImpl {
|
|
|
3726
4012
|
propagated_event: this.propagatedEvent
|
|
3727
4013
|
}).toStr();
|
|
3728
4014
|
}
|
|
4015
|
+
async permalink() {
|
|
4016
|
+
return await permalink(await this.export(), {
|
|
4017
|
+
state: this.state
|
|
4018
|
+
});
|
|
4019
|
+
}
|
|
3729
4020
|
async flush() {
|
|
3730
4021
|
return await this.state.bgLogger().flush();
|
|
3731
4022
|
}
|
|
@@ -3875,15 +4166,17 @@ var Dataset = class extends ObjectFetcher {
|
|
|
3875
4166
|
}) {
|
|
3876
4167
|
this.validateEvent({ metadata, expected, output, tags });
|
|
3877
4168
|
const rowId = id || (0, import_uuid.v4)();
|
|
3878
|
-
const args = this.createArgs(
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
4169
|
+
const args = this.createArgs(
|
|
4170
|
+
deepCopyEvent({
|
|
4171
|
+
id: rowId,
|
|
4172
|
+
input,
|
|
4173
|
+
expected,
|
|
4174
|
+
metadata,
|
|
4175
|
+
tags,
|
|
4176
|
+
output,
|
|
4177
|
+
isMerge: false
|
|
4178
|
+
})
|
|
4179
|
+
);
|
|
3887
4180
|
this.state.bgLogger().log([args]);
|
|
3888
4181
|
return rowId;
|
|
3889
4182
|
}
|
|
@@ -3908,14 +4201,16 @@ var Dataset = class extends ObjectFetcher {
|
|
|
3908
4201
|
id
|
|
3909
4202
|
}) {
|
|
3910
4203
|
this.validateEvent({ metadata, expected, tags });
|
|
3911
|
-
const args = this.createArgs(
|
|
3912
|
-
|
|
3913
|
-
|
|
3914
|
-
|
|
3915
|
-
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
|
|
4204
|
+
const args = this.createArgs(
|
|
4205
|
+
deepCopyEvent({
|
|
4206
|
+
id,
|
|
4207
|
+
input,
|
|
4208
|
+
expected,
|
|
4209
|
+
metadata,
|
|
4210
|
+
tags,
|
|
4211
|
+
isMerge: true
|
|
4212
|
+
})
|
|
4213
|
+
);
|
|
3919
4214
|
this.state.bgLogger().log([args]);
|
|
3920
4215
|
return id;
|
|
3921
4216
|
}
|
|
@@ -3970,7 +4265,7 @@ var Dataset = class extends ObjectFetcher {
|
|
|
3970
4265
|
return await this.state.bgLogger().flush();
|
|
3971
4266
|
}
|
|
3972
4267
|
/**
|
|
3973
|
-
* This function is deprecated. You can simply remove it from your code.
|
|
4268
|
+
* @deprecated This function is deprecated. You can simply remove it from your code.
|
|
3974
4269
|
*/
|
|
3975
4270
|
async close() {
|
|
3976
4271
|
console.warn(
|
|
@@ -5371,7 +5666,10 @@ function parseFilters(filters) {
|
|
|
5371
5666
|
function evaluateFilter(object, filter2) {
|
|
5372
5667
|
const { path: path9, pattern } = filter2;
|
|
5373
5668
|
const key = path9.reduce(
|
|
5374
|
-
(acc, p) => typeof acc === "object" && acc !== null ?
|
|
5669
|
+
(acc, p) => typeof acc === "object" && acc !== null ? (
|
|
5670
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
5671
|
+
acc[p]
|
|
5672
|
+
) : void 0,
|
|
5375
5673
|
object
|
|
5376
5674
|
);
|
|
5377
5675
|
if (key === void 0) {
|
|
@@ -5478,7 +5776,12 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
5478
5776
|
}
|
|
5479
5777
|
);
|
|
5480
5778
|
rootSpan.log({ output, metadata });
|
|
5481
|
-
const scoringArgs = {
|
|
5779
|
+
const scoringArgs = {
|
|
5780
|
+
input: datum.input,
|
|
5781
|
+
expected: "expected" in datum ? datum.expected : void 0,
|
|
5782
|
+
metadata,
|
|
5783
|
+
output
|
|
5784
|
+
};
|
|
5482
5785
|
const scorerNames = evaluator.scores.map(scorerName);
|
|
5483
5786
|
const scoreResults = await Promise.all(
|
|
5484
5787
|
evaluator.scores.map(async (score, score_idx) => {
|
|
@@ -5605,7 +5908,13 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
5605
5908
|
event: {
|
|
5606
5909
|
input: datum.input,
|
|
5607
5910
|
expected: "expected" in datum ? datum.expected : void 0,
|
|
5608
|
-
tags: datum.tags
|
|
5911
|
+
tags: datum.tags,
|
|
5912
|
+
origin: experiment.dataset && datum.id && datum._xact_id ? {
|
|
5913
|
+
object_type: "dataset",
|
|
5914
|
+
object_id: await experiment.dataset.id,
|
|
5915
|
+
id: datum.id,
|
|
5916
|
+
_xact_id: datum._xact_id
|
|
5917
|
+
} : void 0
|
|
5609
5918
|
}
|
|
5610
5919
|
});
|
|
5611
5920
|
}
|
|
@@ -5965,6 +6274,7 @@ function configureNode() {
|
|
|
5965
6274
|
isomorph_default.pathDirname = path.dirname;
|
|
5966
6275
|
isomorph_default.mkdir = fs.mkdir;
|
|
5967
6276
|
isomorph_default.writeFile = fs.writeFile;
|
|
6277
|
+
isomorph_default.readFile = fs.readFile;
|
|
5968
6278
|
_internalSetInitialState();
|
|
5969
6279
|
}
|
|
5970
6280
|
|
|
@@ -6221,7 +6531,7 @@ async function getTsModule() {
|
|
|
6221
6531
|
} catch (e) {
|
|
6222
6532
|
console.warn(
|
|
6223
6533
|
warning(
|
|
6224
|
-
"Failed to load TypeScript module. Will not use
|
|
6534
|
+
"Failed to load TypeScript module. Will not use TypeScript to derive preview."
|
|
6225
6535
|
)
|
|
6226
6536
|
);
|
|
6227
6537
|
}
|
|
@@ -6302,7 +6612,8 @@ async function uploadHandleBundles({
|
|
|
6302
6612
|
}
|
|
6303
6613
|
})
|
|
6304
6614
|
);
|
|
6305
|
-
prompt_data.tool_functions =
|
|
6615
|
+
prompt_data.tool_functions = // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
6616
|
+
resolvableToolFunctions;
|
|
6306
6617
|
}
|
|
6307
6618
|
prompts.push({
|
|
6308
6619
|
project_id: await resolveProjectId(prompt.project),
|
|
@@ -6481,6 +6792,7 @@ async function uploadBundles({
|
|
|
6481
6792
|
const sourceMapContext = await sourceMapContextPromise;
|
|
6482
6793
|
const functionEntries = [
|
|
6483
6794
|
...prompts,
|
|
6795
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
6484
6796
|
...await Promise.all(
|
|
6485
6797
|
bundleSpecs.map(async (spec) => ({
|
|
6486
6798
|
project_id: spec.project_id,
|