braintrust 0.0.21 → 0.0.22

package/dist/logger.js

@@ -0,0 +1,652 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Experiment = exports.summarize = exports.log = exports.login = exports.init = exports.Project = void 0;
+const http = __importStar(require("http"));
+const https = __importStar(require("https"));
+const axios_1 = __importDefault(require("axios"));
+const uuid_1 = require("uuid");
+const git = __importStar(require("./gitutil"));
+const oai_1 = require("./oai");
+let _state = {
+    current_project: null,
+    current_experiment: null,
+};
+let API_URL = null;
+let ORG_ID = null;
+let ORG_NAME = null;
+let LOG_URL = null;
+let LOGGED_IN = false;
+class HTTPConnection {
+    constructor(base_url) {
+        this.base_url = base_url;
+        this.token = null;
+        this.session = null;
+        this._reset();
+    }
+    ping() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const resp = yield this.get("ping");
+                if (_var_user_info === null) {
+                    _var_user_info = resp.data;
+                }
+                return resp.status === 200;
+            }
+            catch (e) {
+                return false;
+            }
+        });
+    }
+    make_long_lived() {
+        // Following a suggestion in https://stackoverflow.com/questions/23013220/max-retries-exceeded-with-url-in-requests
+        this._reset();
+    }
+    set_token(token) {
+        token = token.trim();
+        this.token = token;
+        this._reset();
+    }
+    // As far as I can tell, you cannot set the retry/backoff factor here
+    _reset() {
+        // From https://github.com/axios/axios/issues/1846
+        const httpAgent = new http.Agent({ keepAlive: true });
+        const httpsAgent = new https.Agent({ keepAlive: true });
+        let headers = {};
+        if (this.token) {
+            headers["Authorization"] = `Bearer ${this.token}`;
+        }
+        this.session = axios_1.default.create({
+            httpAgent,
+            httpsAgent,
+            headers,
+        });
+    }
+    get(path, params = undefined) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return yield this.session.get(_urljoin(this.base_url, path), { params });
+        });
+    }
+    post(path, params = undefined) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return yield this.session.post(_urljoin(this.base_url, path), params);
+        });
+    }
+}
+let _api_conn = null;
+function api_conn() {
+    if (!_api_conn) {
+        _api_conn = new HTTPConnection(LOG_URL);
+    }
+    return _api_conn;
+}
+function api_get(object_type, args = undefined, retries = 0) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const tries = retries + 1;
+        for (let i = 0; i < tries; i++) {
+            try {
+                const resp = yield api_conn().get(`${object_type}`, args);
+                return resp.data;
+            }
+            catch (e) {
+                if (i < tries - 1) {
+                    console.log(`Retrying API request ${object_type} ${args} ${e.status} ${e.text}`);
+                    continue;
+                }
+                throw e;
+            }
+        }
+    });
+}
+function api_insert(object_type, args = undefined) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const resp = yield api_conn().post(`${object_type}`, args);
+        return resp.data;
+    });
+}
+let _var_user_info = null;
+function _user_info() {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (_var_user_info === null) {
+            _var_user_info = yield api_get("ping");
+        }
+        return _var_user_info;
+    });
+}
+class Project {
+    constructor(name, id, org_id) {
+        this.name = name;
+        this.id = id;
+        this.org_id = org_id;
+    }
+}
+exports.Project = Project;
+// NOTE: This is because we do not have async constructors
+const _PROJECTS_ENDPOINT = "projects";
+function initProject(name) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const unique_key = { name, org_id: ORG_ID };
+        // Can we have an upsert (or insert if not exists) method instead?
+        let existing = [];
+        if (unique_key) {
+            existing = yield api_get(_PROJECTS_ENDPOINT, unique_key);
+        }
+        if (existing.length === 0) {
+            existing = yield api_insert(_PROJECTS_ENDPOINT, unique_key);
+        }
+        if (existing) {
+            return existing[0];
+        }
+        else {
+            throw new Error(`Unable to find record in ${_PROJECTS_ENDPOINT}`);
+        }
+    });
+}
+/*
+def guess_git_experiment_name():
+    try:
+        repo = git.Repo(search_parent_directories=True)
+    except git.InvalidGitRepositoryError:
+        return None
+
+    branch = repo.active_branch.name
+    diff = repo.git.diff(repo.head.commit.tree)
+    if not diff and len(repo.head.commit.parents) > 0:
+        diff = repo.head.commit.message + "\n" + repo.git.diff(repo.head.commit.tree, repo.head.commit.parents[0].tree)
+
+    return [
+        {
+            "role": "system",
+            "content": """\
+You can generate two word summaries for machine learning experiment names, based
+on the branch name and an optional "diff" of the experiment's code on top of the branch.
+The experiment name should be exactly two words, concatenated with a hyphen, all lowercase.
+The input format is the output of "git diff". For example, "foo-bar" is valid but
+"foo-bar-baz" is not.""",
+        },
+        {
+            "role": "user",
+            "content": f"Branch: {branch}" + (f"\n\nDiff:\n{diff[:4096]}" if diff else ""),
+        },
+    ]
+*/
+function guessGitExperimentName() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const repo = yield git.currentRepo();
+        if (!repo) {
+            return undefined;
+        }
+        const branch = yield repo.raw(["rev-parse", "--abbrev-ref", "HEAD"]);
+        let diff = yield repo.diff();
+        if (!diff) {
+            const last_commit = yield repo.log({ maxCount: 2 });
+            if (last_commit.all.length > 1) {
+                const parent = last_commit.all[1];
+                diff = parent.message + "\n" + (yield repo.diff(["HEAD", parent.hash]));
+            }
+        }
+        return [
+            {
+                role: "system",
+                content: `You can generate two word summaries for machine learning experiment names, based
+on the branch name and an optional "diff" of the experiment's code on top of the branch.
+The experiment name should be exactly two words, concatenated with a hyphen, all lowercase.
+The input format is the output of "git diff". For example, "foo-bar" is valid but
+"foo-bar-baz" is not.`,
+            },
+            {
+                role: "user",
+                content: `Branch: ${branch}` + (diff ? `\n\nDiff:\n${diff.slice(0, 4096)}` : ""),
+            },
+        ];
+    });
+}
+function guessExperimentName() {
+    return __awaiter(this, void 0, void 0, function* () {
+        if ((0, oai_1.openAI)() === null) {
+            return undefined;
+        }
+        const messages = yield guessGitExperimentName();
+        if (!messages) {
+            return undefined;
+        }
+        const resp = yield (0, oai_1.cachedChatCompletion)({
+            model: "gpt-3.5-turbo",
+            messages,
+            max_tokens: 128,
+            temperature: 0.7,
+        });
+        let name = undefined;
+        if (resp.choices.length > 0) {
+            name = resp.choices[0].message.content.split("-").slice(0, 2).join("-");
+            // Strip punctuation and whitespace from the prefix and suffix
+            name = name === null || name === void 0 ? void 0 : name.replace(/^[ .,;:!?-]+|[ .,;:!?-]+$/g, "");
+        }
+        return name;
+    });
+}
+class LogThread {
+    constructor() {
+        this.items = [];
+        this.active_flush = Promise.resolve([]);
+        this.active_flush_resolved = true;
+    }
+    log(items) {
+        this.items.push(...items);
+        if (this.active_flush_resolved) {
+            this.active_flush_resolved = false;
+            this.active_flush = this.flush_once();
+        }
+    }
+    flush_once() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.active_flush_resolved = false;
+            const items = this.items;
+            this.items = [];
+            let ret = [];
+            if (items.length > 0) {
+                const resp = yield api_insert("logs", items);
+                ret = resp.data;
+            }
+            // If more items were added while we were flushing, flush again
+            if (this.items.length > 0) {
+                this.active_flush = this.flush_once();
+            }
+            else {
+                this.active_flush_resolved = true;
+            }
+            return ret;
+        });
+    }
+    flush() {
+        return __awaiter(this, void 0, void 0, function* () {
+            while (true) {
+                yield this.active_flush;
+                if (this.active_flush_resolved) {
+                    break;
+                }
+            }
+        });
+    }
+}
+/**
+ * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
+ *
+ * @param project The name of the project to create the experiment in.
+ * @param options Additional options for configuring init().
+ * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
+ * @param options.description An optional description of the experiment.
+ * @param options.update If the experiment already exists, continue logging to it.
+ * @param options.baseExperiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
+ * experiment. Otherwise, it will pick an experiment by finding the closest ancestor on the default (e.g. main) branch.
+ * @param options.apiUrl The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disableCache Do not use cached login information.
+ * @returns The newly created Experiment.
+ */
+function init(project, options = {}) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { experiment, description, baseExperiment, update, apiUrl, apiKey, orgName, disableCache, } = options || {};
+        yield login({
+            orgName: orgName,
+            disableCache,
+            apiKey,
+            apiUrl,
+        });
+        _state.current_project = yield initProject(project);
+        _state.current_experiment = yield initExperiment(_state.current_project, {
+            name: experiment,
+            description,
+            update,
+            baseExperiment,
+        });
+        return _state.current_experiment;
+    });
+}
+exports.init = init;
+/**
+ * Log into BrainTrust. This will prompt you for your API token, which you can find at
+ * https://www.braintrustdata.com/app/token. This method is called automatically by `init()`.
+ *
+ * @param options Options for configuring login().
+ * @param options.apiUrl The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disableCache Do not use cached login information.
+ * @param options.forceLogin Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
+ */
+function login(options = {}) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { apiUrl = process.env.BRAINTRUST_API_URL || "https://www.braintrustdata.com", apiKey = process.env.BRAINTRUST_API_KEY, orgName: orgName = undefined, disableCache = false, forceLogin = false, } = options || {};
+        if (LOGGED_IN && !forceLogin) {
+            return;
+        }
+        API_URL = apiUrl;
+        let login_key_info = null;
+        let ping_ok = false;
+        let conn = null;
+        if (apiKey !== undefined) {
+            const resp = yield axios_1.default.post(_urljoin(API_URL, `/api/apikey/login`), {
+                token: apiKey,
+            });
+            const info = resp.data;
+            _check_org_info(info.org_info, orgName);
+            conn = api_conn();
+            conn.set_token(apiKey);
+            ping_ok = yield conn.ping();
+        }
+        else {
+            // TODO: Implement token based login in the JS client
+            throw new Error("Please specify an api key. Token based login is not yet implemented in the JS client.");
+        }
+        if (!conn) {
+            throw new Error("Conn should be set at this point (a bug)");
+        }
+        if (!ping_ok) {
+            yield conn.get("ping");
+        }
+        conn.make_long_lived();
+        LOGGED_IN = true;
+    });
+}
+exports.login = login;
+/**
+ * Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
+ *
+ * @param event The event to log.
+ * @param event.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+ * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+ * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+ * `inputs` should be identical.
+ * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+ * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+ * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+ * be multiple valid queries that answer a single question.
+ * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+ * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+ * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+ * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+ * fine-tune your models.
+ * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+ * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+ * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+ * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+ * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+ * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+ * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+ * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+ * JSON-serializable type, but its keys must be strings.
+ * @param event.id (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
+ * @returns The `id` of the logged event.
+ */
+function log(options) {
+    if (!_state.current_experiment) {
+        throw new Error("Not initialized. Please call init() first");
+    }
+    return _state.current_experiment.log(options);
+}
+exports.log = log;
+/**
+ * Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
+ *
+ * @param options Options for summarizing the experiment.
+ * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
+ * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+ * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
+ */
+function summarize(options = {}) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!_state.current_experiment) {
+            throw new Error("Not initialized. Please call init() first");
+        }
+        return yield _state.current_experiment.summarize(options);
+    });
+}
+exports.summarize = summarize;
+function _check_org_info(org_info, org_name) {
+    if (org_info.length === 0) {
+        throw new Error("This user is not part of any organizations.");
+    }
+    for (const org of org_info) {
+        if (org_name === undefined || org.name === org_name) {
+            ORG_ID = org.id;
+            ORG_NAME = org.name;
+            LOG_URL = org.api_url;
+            break;
+        }
+    }
+    if (ORG_ID === undefined) {
+        throw new Error(`Organization ${org_name} not found. Must be one of ${org_info
+            .map((x) => x.name)
+            .join(", ")}`);
+    }
+}
+function _urljoin(...parts) {
+    return parts.map((x) => x.replace(/^\//, "")).join("/");
+}
+function initExperiment(project, { name, description, update, baseExperiment, } = {
+    name: undefined,
+    description: undefined,
+    baseExperiment: undefined,
+}) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const args = { project_id: project.id };
+        if (!name) {
+            name = yield guessExperimentName();
+        }
+        if (name) {
+            args["name"] = name;
+        }
+        if (description) {
+            args["description"] = description;
+        }
+        if (update) {
+            args["update"] = update;
+        }
+        const repoStatus = yield git.getRepoStatus();
+        if (repoStatus) {
+            args["repo_info"] = repoStatus;
+        }
+        const conn = api_conn();
+        let base_exp_id = undefined;
+        if (baseExperiment !== undefined) {
+            const resp = yield conn.get("experiments", {
+                project_id: project.id,
+                name: baseExperiment,
+            });
+            const experiments = resp.data;
+            if (experiments.length > 0) {
+                base_exp_id = experiments[0]["id"];
+            }
+            else {
+                throw new Error(`Base experiment ${baseExperiment} not found`);
+            }
+        }
+        if (base_exp_id === undefined) {
+            const resp = yield conn.post("experiments-by-commits", {
+                project_id: project.id,
+                commits: yield git.getPastNAncestors(),
+            });
+            base_exp_id = resp.data["experiment_id"];
+        }
+        if (base_exp_id !== undefined) {
+            args["base_exp_id"] = base_exp_id;
+        }
+        const data = (yield api_insert("register-experiment", args))[0];
+        // NOTE: This is a deviation from the Python lib and allows the log() method
+        // to not be async.
+        //
+        const user_id = (yield _user_info())["id"];
+        return new Experiment(project, data.id, data.name, user_id);
+    });
+}
+/**
+ * An experiment is a collection of logged events, such as model inputs and outputs, which represent
+ * a snapshot of your application at a particular point in time. An experiment is meant to capture more
+ * than just the model you use, and includes the data you use to test, pre- and post- processing code,
+ * comparison metrics (scores), and any other metadata you want to include.
+ *
+ * Experiments are associated with a project, and two experiments are meant to be easily comparable via
+ * their `inputs`. You can change the attributes of the experiments in a project (e.g. scoring functions)
+ * over time, simply by changing what you log.
+ *
+ * You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
+ */
+class Experiment {
+    constructor(project, id, name, user_id) {
+        this.project = project;
+        this.id = id;
+        this.name = name;
+        this.user_id = user_id;
+        this.logger = new LogThread();
+    }
+    /**
+     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+     *
+     * @param event The event to log.
+     * @param event.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+     * `inputs` should be identical.
+     * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+     * be multiple valid queries that answer a single question.
+     * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+     * fine-tune your models.
+     * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+     * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+     * JSON-serializable type, but its keys must be strings.
+     * @param event.id (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
+     * @returns The `id` of the logged event.
+     */
+    log({ inputs, output, expected, scores, metadata, id, }) {
+        for (const [name, score] of Object.entries(scores)) {
+            if (typeof name !== "string") {
+                throw new Error("score names must be strings");
+            }
+            if (typeof score !== "number") {
+                throw new Error("score values must be numbers");
+            }
+            if (score < 0 || score > 1) {
+                throw new Error("score values must be between 0 and 1");
+            }
+        }
+        if (metadata !== undefined) {
+            for (const key of Object.keys(metadata)) {
+                if (typeof key !== "string") {
+                    throw new Error("metadata keys must be strings");
+                }
+            }
+        }
+        const args = {
+            id: id || (0, uuid_1.v4)(),
+            inputs,
+            output,
+            expected,
+            scores,
+            project_id: this.project.id,
+            experiment_id: this.id,
+            user_id: this.user_id,
+            created: new Date().toISOString(),
+            metadata,
+        };
+        if (JSON.stringify(args).length > 65535) {
+            throw new Error("Events must be less than 64KB in size. Please reduce the size of your inputs, output, expected, scores, or metadata.");
+        }
+        this.logger.log([args]);
+        return args.id;
+    }
+    /**
+     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     *
+     * @param options Options for summarizing the experiment.
+     * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     */
+    summarize(options = {}) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { summarizeScores = true, comparisonExperimentId = undefined } = options || {};
+            yield this.logger.flush();
+            const projectUrl = `${API_URL}/app/${encodeURIComponent(ORG_NAME)}/p/${encodeURIComponent(this.project.name)}`;
+            const experimentUrl = `${projectUrl}/${encodeURIComponent(this.name)}`;
+            let scores = undefined;
+            let comparisonExperimentName = undefined;
+            if (summarizeScores) {
+                if (comparisonExperimentId === undefined) {
+                    const conn = api_conn();
+                    const resp = yield conn.get("/crud/base_experiments", {
+                        id: this.id,
+                    });
+                    const base_experiments = resp.data;
+                    if (base_experiments.length > 0) {
+                        comparisonExperimentId = base_experiments[0]["base_exp_id"];
+                        comparisonExperimentName = base_experiments[0]["base_exp_name"];
+                    }
+                }
+                if (comparisonExperimentId !== undefined) {
+                    scores = yield api_get("/experiment-comparison", {
+                        experiment_id: this.id,
+                        base_experiment_id: comparisonExperimentId,
+                    }, 3);
+                }
+            }
+            return {
+                projectName: this.project.name,
+                experimentName: this.name,
+                projectUrl: projectUrl,
+                experimentUrl: experimentUrl,
+                comparisonExperimentName: comparisonExperimentName,
+                scores,
+            };
+        });
+    }
+}
+exports.Experiment = Experiment;

package/dist/oai.js

@@ -1,4 +1,6 @@
 "use strict";
+// TODO REPLACE WITH autoevals version
+// https://github.com/braintrustdata/braintrust/issues/218
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);