braintrust 0.0.2 → 0.0.4

package/README.md

@@ -0,0 +1,30 @@
+A Node.js library for logging data to BrainTrust.
+
+### Quickstart
+
+Install the library with npm (or yarn).
+
+```bash
+npm install braintrust
+```
+
+Then, run a simple experiment with the following code (replace `YOUR_API_KEY` with
+your BrainTrust API key):
+
+```javascript
+const braintrust = require("braintrust");
+
+const experiment = await braintrust.init("NodeTest", {api_key: "YOUR_API_KEY"});
+experiment.log({
+  inputs: {test: 1},
+  output: "foo",
+  expected: "bar",
+  scores: {
+    n: 0.5,
+  },
+  metadata: {
+    id: 1,
+  },
+});
+console.log(await experiment.summarize());
+```

package/dist/api.js

@@ -0,0 +1 @@
+"use strict";

package/dist/dep.d.ts

@@ -0,0 +1,52 @@
+export declare class Experiment {
+    private _project;
+    constructor(project: string);
+    get project(): string;
+    /**
+     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+     *
+     * @param values
+     * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+     * `inputs` should be identical.
+     * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+     * be multiple valid queries that answer a single question.
+     * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+     * fine-tune your models.
+     * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+     * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+     * JSON-serializable type, but its keys must be strings.
+     * @returns The `id` of the logged event.
+     */
+    log({ inputs, output, expected, scores, metadata, }: {
+        readonly inputs: unknown;
+        readonly output: unknown;
+        readonly expected: unknown;
+        readonly scores: Record<string, number>;
+        readonly metadata?: Record<string, unknown>;
+    }): string;
+    /**
+     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     *
+     * @param options
+     * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns `ExperimentSummary`
+     */
+    summarize(options?: {
+        readonly summarizeScores?: boolean;
+        readonly comparisonExperimentId?: string;
+    } | undefined): string;
+}

package/dist/dep.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Experiment = void 0;
+class Experiment {
+    constructor(project) {
+        this._project = project;
+    }
+    get project() {
+        return this._project;
+    }
+    /**
+     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+     *
+     * @param values
+     * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+     * `inputs` should be identical.
+     * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+     * be multiple valid queries that answer a single question.
+     * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+     * fine-tune your models.
+     * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+     * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+     * JSON-serializable type, but its keys must be strings.
+     * @returns The `id` of the logged event.
+     */
+    log({ inputs, output, expected, scores, metadata, }) {
+        // TODO
+        (() => ({ inputs, output, expected, scores, metadata }))();
+        return "foo";
+    }
+    /**
+     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     *
+     * @param options
+     * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns `ExperimentSummary`
+     */
+    summarize(options = undefined) {
+        return "Summary!";
+    }
+}
+exports.Experiment = Experiment;

package/dist/index.d.ts

@@ -1,8 +1,48 @@
+/**
+ * A Node.js library for logging data to BrainTrust.
+ *
+ * ### Quickstart
+ *
+ * Install the library with npm (or yarn).
+ *
+ * ```bash
+ * npm install braintrust
+ * ```
+ *
+ * Then, run a simple experiment with the following code (replace `YOUR_API_KEY` with
+ * your BrainTrust API key):
+ *
+ * ```javascript
+ * const braintrust = require("braintrust");
+ *
+ * const experiment = await braintrust.init("NodeTest", {api_key: "YOUR_API_KEY"});
+ * experiment.log({
+ *   inputs: {test: 1},
+ *   output: "foo",
+ *   expected: "bar",
+ *   scores: {
+ *     n: 0.5,
+ *   },
+ *   metadata: {
+ *     id: 1,
+ *   },
+ * });
+ * console.log(await experiment.summarize());
+ * ```
+ *
+ * @module braintrust
+ */
+export declare class Project {
+    name: string;
+    id: string;
+    org_id: string;
+    constructor(name: string, id: string, org_id: string);
+}
 /**
  * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
  *
  * @param project The name of the project to create the experiment in.
- * @param options
+ * @param options Additional options for configuring init().
  * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
  * @param options.description An optional description of the experiment.
  * @param options.base_experiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
@@ -14,7 +54,7 @@
  * @param options.disable_cache Do not use cached login information.
  * @returns The experiment object.
  */
-export declare function init(project: string, { experiment, }: {
+export declare function init(project: string, options?: {
     readonly experiment?: string;
     readonly description?: string;
     readonly base_experiment?: string;
@@ -22,11 +62,98 @@ export declare function init(project: string, { experiment, }: {
     readonly api_key?: string;
     readonly org_name?: string;
     readonly disable_cache?: boolean;
-}): Experiment;
+}): Promise<Experiment>;
+/**
+ * Log into BrainTrust. This will prompt you for your API token, which you can find at
+ * https://www.braintrustdata.com/app/token. This method is called automatically by `init()`.
+ *
+ * @param options
+ * @param options.api_url The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.api_key The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.org_name (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disable_cache Do not use cached login information.
+ * @param options.force_login Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
+ * @returns
+ */
+export declare function login(options?: {
+    api_url?: string;
+    api_key?: string;
+    org_name?: string;
+    disable_cache?: boolean;
+    force_login?: boolean;
+} | undefined): Promise<void>;
+/**
+ * Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
+ *
+ * @param values
+ * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+ * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+ * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+ * `inputs` should be identical.
+ * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+ * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+ * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+ * be multiple valid queries that answer a single question.
+ * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+ * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+ * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+ * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+ * fine-tune your models.
+ * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+ * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+ * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+ * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+ * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+ * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+ * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+ * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+ * JSON-serializable type, but its keys must be strings.
+ * @returns The `id` of the logged event.
+ */
+export declare function log(options: {
+    readonly inputs: unknown;
+    readonly output: unknown;
+    readonly expected: unknown;
+    readonly scores: Record<string, number>;
+    readonly metadata?: Record<string, unknown>;
+}): string;
+/**
+ * Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
+ *
+ * @param options
+ * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+ * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+ * @returns `ExperimentSummary`
+ */
+export declare function summarize(options?: {
+    readonly summarizeScores?: boolean;
+    readonly comparisonExperimentId?: string;
+} | undefined): Promise<ExperimentSummary>;
+/**
+ * An experiment is a collection of logged events, such as model inputs and outputs, which represent
+ * a snapshot of your application at a particular point in time. An experiment is meant to capture more
+ * than just the model you use, and includes the data you use to test, pre- and post- processing code,
+ * comparison metrics (scores), and any other metadata you want to include.
+ *
+ * Experiments are associated with a project, and two experiments are meant to be easily comparable via
+ * their `inputs`. You can change the attributes of the experiments in a project (e.g. scoring functions)
+ * over time, simply by changing what you log.
+ *
+ * You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
+ */
 export declare class Experiment {
-    private _project;
-    constructor(project: string);
-    get project(): string;
+    readonly project: Project;
+    readonly id: string;
+    readonly name: string;
+    readonly user_id: string;
+    private logger;
+    constructor(project: Project, id: string, name: string, user_id: string);
+    static init(project: Project, { name, description, base_experiment, }?: {
+        name?: string;
+        description?: string;
+        base_experiment?: string;
+    }): Promise<Experiment>;
     /**
      * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
      *
@@ -73,5 +200,38 @@ export declare class Experiment {
     summarize(options?: {
         readonly summarizeScores?: boolean;
         readonly comparisonExperimentId?: string;
-    } | undefined): string;
+    } | undefined): Promise<ExperimentSummary>;
+}
+/**
+ * Summary of a score's performance.
+ * @property name Name of the score.
+ * @property score Average score across all examples.
+ * @property diff Difference in score between the current and reference experiment.
+ * @property improvements Number of improvements in the score.
+ * @property regressions Number of regressions in the score.
+ */
+interface ScoreSummary {
+    name: string;
+    score: number;
+    diff: number;
+    improvements: number;
+    regressions: number;
+}
+/**
+ * Summary of an experiment's scores and metadata.
+ * @property projectName Name of the project that the experiment belongs to.
+ * @property experimentName Name of the experiment.
+ * @property projectUrl URL to the project's page in the BrainTrust app.
+ * @property experimentUrl URL to the experiment's page in the BrainTrust app.
+ * @property comparisonExperimentName The experiment scores are baselined against.
+ * @property scores Summary of the experiment's scores.
+ */
+interface ExperimentSummary {
+    projectName: string;
+    experimentName: string;
+    projectUrl: string;
+    experimentUrl: string;
+    comparisonExperimentName: string | undefined;
+    scores: Record<string, ScoreSummary> | undefined;
 }
+export {};

package/dist/index.js

@@ -1,9 +1,312 @@
 "use strict";
+/**
+ * A Node.js library for logging data to BrainTrust.
+ *
+ * ### Quickstart
+ *
+ * Install the library with npm (or yarn).
+ *
+ * ```bash
+ * npm install braintrust
+ * ```
+ *
+ * Then, run a simple experiment with the following code (replace `YOUR_API_KEY` with
+ * your BrainTrust API key):
+ *
+ * ```javascript
+ * const braintrust = require("braintrust");
+ *
+ * const experiment = await braintrust.init("NodeTest", {api_key: "YOUR_API_KEY"});
+ * experiment.log({
+ *   inputs: {test: 1},
+ *   output: "foo",
+ *   expected: "bar",
+ *   scores: {
+ *     n: 0.5,
+ *   },
+ *   metadata: {
+ *     id: 1,
+ *   },
+ * });
+ * console.log(await experiment.summarize());
+ * ```
+ *
+ * @module braintrust
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Experiment = exports.summarize = exports.log = exports.login = exports.init = exports.Project = void 0;
+const http = __importStar(require("http"));
+const https = __importStar(require("https"));
+const axios_1 = __importDefault(require("axios"));
+const uuid_1 = require("uuid");
+let _state = {
+    current_project: null,
+    current_experiment: null,
+};
+let API_URL = null;
+let ORG_ID = null;
+let ORG_NAME = null;
+let LOG_URL = null;
+let LOGGED_IN = false;
+class HTTPConnection {
+    constructor(base_url) {
+        this.base_url = base_url;
+        this.token = null;
+        this.session = null;
+        this._reset();
+    }
+    ping() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const resp = yield this.get("ping");
+                if (_var_user_info === null) {
+                    _var_user_info = resp.data;
+                }
+                return resp.status === 200;
+            }
+            catch (e) {
+                return false;
+            }
+        });
+    }
+    make_long_lived() {
+        // Following a suggestion in https://stackoverflow.com/questions/23013220/max-retries-exceeded-with-url-in-requests
+        this._reset();
+    }
+    set_token(token) {
+        token = token.trim();
+        this.token = token;
+        this._reset();
+    }
+    // As far as I can tell, you cannot set the retry/backoff factor here
+    _reset() {
+        // From https://github.com/axios/axios/issues/1846
+        const httpAgent = new http.Agent({ keepAlive: true });
+        const httpsAgent = new https.Agent({ keepAlive: true });
+        let headers = {};
+        if (this.token) {
+            headers["Authorization"] = `Bearer ${this.token}`;
+        }
+        this.session = axios_1.default.create({
+            httpAgent,
+            httpsAgent,
+            headers,
+        });
+    }
+    get(path, params = undefined) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return yield this.session.get(_urljoin(this.base_url, path), { params });
+        });
+    }
+    post(path, params = undefined) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return yield this.session.post(_urljoin(this.base_url, path), params);
+        });
+    }
+}
+let _api_conn = null;
+function api_conn() {
+    if (!_api_conn) {
+        _api_conn = new HTTPConnection(LOG_URL);
+    }
+    return _api_conn;
+}
+function api_get(object_type, args = undefined) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const resp = yield api_conn().get(`${object_type}`, args);
+        return resp.data;
+    });
+}
+function api_insert(object_type, args = undefined) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const resp = yield api_conn().post(`${object_type}`, args);
+        return resp.data;
+    });
+}
+let _var_user_info = null;
+function _user_info() {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (_var_user_info === null) {
+            _var_user_info = yield api_get("ping");
+        }
+        return _var_user_info;
+    });
+}
+class Project {
+    constructor(name, id, org_id) {
+        this.name = name;
+        this.id = id;
+        this.org_id = org_id;
+    }
+}
+exports.Project = Project;
+// NOTE: This is because we do not have async constructors
+const _PROJECTS_ENDPOINT = "projects";
+function initProject(name) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const unique_key = { name, org_id: ORG_ID };
+        // Can we have an upsert (or insert if not exists) method instead?
+        let existing = [];
+        if (unique_key) {
+            existing = yield api_get(_PROJECTS_ENDPOINT, unique_key);
+        }
+        if (existing.length === 0) {
+            existing = yield api_insert(_PROJECTS_ENDPOINT, unique_key);
+        }
+        if (existing) {
+            return existing[0];
+        }
+        else {
+            throw new Error(`Unable to find record in ${_PROJECTS_ENDPOINT}`);
+        }
+    });
+}
+/*
+def guess_git_experiment_name():
+    try:
+        repo = git.Repo(search_parent_directories=True)
+    except git.InvalidGitRepositoryError:
+        return None
+
+    branch = repo.active_branch.name
+    diff = repo.git.diff(repo.head.commit.tree)
+    if not diff and len(repo.head.commit.parents) > 0:
+        diff = repo.head.commit.message + "\n" + repo.git.diff(repo.head.commit.tree, repo.head.commit.parents[0].tree)
+
+    return [
+        {
+            "role": "system",
+            "content": """\
+You can generate two word summaries for machine learning experiment names, based
+on the branch name and an optional "diff" of the experiment's code on top of the branch.
+The experiment name should be exactly two words, concatenated with a hyphen, all lowercase.
+The input format is the output of "git diff". For example, "foo-bar" is valid but
+"foo-bar-baz" is not.""",
+        },
+        {
+            "role": "user",
+            "content": f"Branch: {branch}" + (f"\n\nDiff:\n{diff[:4096]}" if diff else ""),
+        },
+    ]
+
+
+def guess_experiment_name():
+    # OpenAI is very slow to import, so we only do it when we need it
+    import openai
+
+    if openai.api_key is None:
+        return None
+
+    messages = guess_notebook_block_name()
+    if not messages:
+        messages = guess_git_experiment_name()
+
+    if not messages:
+        return None
+
+    resp = run_cached_request(
+        Completion=openai.ChatCompletion,
+        model="gpt-3.5-turbo",
+        messages=messages,
+        max_tokens=128,
+        temperature=0.7,
+    )
+
+    name = None
+    if len(resp["choices"]) > 0:
+        name = "-".join(resp["choices"][0]["message"]["content"].split("-")[:2])
+        # Strip punctuation and whitespace from the prefix and suffix
+        name = name.strip(" .,;:!?-")
+    return name
+*/
+function guess_experiment_name() {
+    // TODO: Implement experiment name guessing from git
+    return null;
+}
+class LogThread {
+    constructor() {
+        this.items = [];
+        this.active_flush = Promise.resolve([]);
+        this.active_flush_resolved = true;
+    }
+    log(items) {
+        this.items.push(...items);
+        if (this.active_flush_resolved) {
+            this.active_flush_resolved = false;
+            this.active_flush = this.flush_once();
+        }
+    }
+    flush_once() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.active_flush_resolved = false;
+            const items = this.items;
+            this.items = [];
+            let ret = [];
+            if (items.length > 0) {
+                const resp = yield api_insert("logs", items);
+                ret = resp.data;
+            }
+            // If more items were added while we were flushing, flush again
+            if (this.items.length > 0) {
+                this.active_flush = this.flush_once();
+            }
+            else {
+                this.active_flush_resolved = true;
+            }
+            return ret;
+        });
+    }
+    flush() {
+        return __awaiter(this, void 0, void 0, function* () {
+            while (true) {
+                yield this.active_flush;
+                if (this.active_flush_resolved) {
+                    break;
+                }
+            }
+        });
+    }
+}
 /**
  * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
  *
  * @param project The name of the project to create the experiment in.
- * @param options
+ * @param options Additional options for configuring init().
  * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
  * @param options.description An optional description of the experiment.
  * @param options.base_experiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
@@ -15,19 +318,226 @@
  * @param options.disable_cache Do not use cached login information.
  * @returns The experiment object.
  */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Experiment = exports.init = void 0;
-function init(project, { experiment, }) {
-    // TODO
-    return new Experiment(project);
+function init(project, options = {}) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { experiment, description, base_experiment, api_url, api_key, org_name, disable_cache, } = options || {};
+        yield login({
+            org_name,
+            disable_cache,
+            api_key,
+            api_url,
+        });
+        _state.current_project = yield initProject(project);
+        _state.current_experiment = yield Experiment.init(_state.current_project, {
+            name: experiment,
+            description,
+            base_experiment,
+        });
+        return _state.current_experiment;
+    });
 }
 exports.init = init;
+/**
+ * Log into BrainTrust. This will prompt you for your API token, which you can find at
+ * https://www.braintrustdata.com/app/token. This method is called automatically by `init()`.
+ *
+ * @param options
+ * @param options.api_url The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.api_key The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.org_name (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disable_cache Do not use cached login information.
+ * @param options.force_login Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
+ * @returns
+ */
+function login(options = undefined) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { api_url = process.env.BRAINTRUST_API_URL ||
+            "https://www.braintrustdata.com", api_key = process.env.BRAINTRUST_API_KEY, org_name = undefined, disable_cache = false, force_login = false, } = options || {};
+        if (LOGGED_IN && !force_login) {
+            return;
+        }
+        API_URL = api_url;
+        let login_key_info = null;
+        let ping_ok = false;
+        let conn = null;
+        if (api_key !== undefined) {
+            const resp = yield axios_1.default.post(_urljoin(API_URL, `/api/apikey/login`), {
+                token: api_key,
+            });
+            const info = resp.data;
+            _check_org_info(info.org_info, org_name);
+            conn = api_conn();
+            conn.set_token(api_key);
+            ping_ok = yield conn.ping();
+        }
+        else {
+            // TODO: Implement token based login in the JS client
+            throw new Error("Please specify an api key. Token based login is not yet implemented in the JS client.");
+        }
+        if (!conn) {
+            throw new Error("Conn should be set at this point (a bug)");
+        }
+        if (!ping_ok) {
+            yield conn.get("ping");
+        }
+        conn.make_long_lived();
+        LOGGED_IN = true;
+    });
+}
+exports.login = login;
+/**
+ * Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
+ *
+ * @param values
+ * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+ * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+ * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+ * `inputs` should be identical.
+ * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+ * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+ * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+ * be multiple valid queries that answer a single question.
+ * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+ * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+ * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+ * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+ * fine-tune your models.
+ * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+ * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+ * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+ * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+ * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+ * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+ * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+ * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+ * JSON-serializable type, but its keys must be strings.
+ * @returns The `id` of the logged event.
+ */
+function log(options) {
+    if (!_state.current_experiment) {
+        throw new Error("Not initialized. Please call init() first");
+    }
+    return _state.current_experiment.log(options);
+}
+exports.log = log;
+/**
+ * Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
+ *
+ * @param options
+ * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+ * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+ * @returns `ExperimentSummary`
+ */
+function summarize(options = undefined) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!_state.current_experiment) {
+            throw new Error("Not initialized. Please call init() first");
+        }
+        return yield _state.current_experiment.summarize(options);
+    });
+}
+exports.summarize = summarize;
+function _check_org_info(org_info, org_name) {
+    if (org_info.length === 0) {
+        throw new Error("This user is not part of any organizations.");
+    }
+    for (const org of org_info) {
+        if (org_name === undefined || org.name === org_name) {
+            ORG_ID = org.id;
+            ORG_NAME = org.name;
+            LOG_URL = org.api_url;
+            break;
+        }
+    }
+    if (ORG_ID === undefined) {
+        throw new Error(`Organization ${org_name} not found. Must be one of ${org_info
+            .map((x) => x.name)
+            .join(", ")}`);
+    }
+}
+function _urljoin(...parts) {
+    return parts.map((x) => x.replace(/^\//, "")).join("/");
+}
+/**
+ * An experiment is a collection of logged events, such as model inputs and outputs, which represent
+ * a snapshot of your application at a particular point in time. An experiment is meant to capture more
+ * than just the model you use, and includes the data you use to test, pre- and post- processing code,
+ * comparison metrics (scores), and any other metadata you want to include.
+ *
+ * Experiments are associated with a project, and two experiments are meant to be easily comparable via
+ * their `inputs`. You can change the attributes of the experiments in a project (e.g. scoring functions)
+ * over time, simply by changing what you log.
+ *
+ * You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
+ */
 class Experiment {
-    constructor(project) {
-        this._project = project;
+    constructor(project, id, name, user_id) {
+        this.project = project;
+        this.id = id;
+        this.name = name;
+        this.user_id = user_id;
+        this.logger = new LogThread();
     }
-    get project() {
-        return this._project;
+    static init(project, { name, description, base_experiment, } = {
+        name: undefined,
+        description: undefined,
+        base_experiment: undefined,
+    }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const args = { project_id: project.id };
+            // TODO
+            /*
+            if (!name) {
+              name = guessExperimentName();
+            }
+            */
+            if (name) {
+                args["name"] = name;
+            }
+            if (description) {
+                args["description"] = description;
+            }
+            // TODO
+            /*
+            const repoStatus = await get_repo_status();
+            if (repoStatus) {
+              args["repo_info"] = repoStatus;
+            }
+            */
+            const conn = api_conn();
+            let base_exp_id = undefined;
+            if (base_experiment !== undefined) {
+                const resp = yield conn.get("experiments", {
+                    project_id: project.id,
+                    name: base_experiment,
+                });
+                const experiments = resp.data;
+                if (experiments.length > 0) {
+                    base_exp_id = experiments[0]["id"];
+                }
+                else {
+                    throw new Error(`Base experiment ${base_experiment} not found`);
+                }
+            }
+            if (base_exp_id === undefined) {
+                const resp = yield conn.post("experiments-by-commits", {
+                    project_id: project.id,
+                    // TODO
+                    commits: [] /* await get_past_n_ancestors() */,
+                });
+                base_exp_id = resp.data["experiment_id"];
+            }
+            if (base_exp_id !== undefined) {
+                args["base_exp_id"] = base_exp_id;
+            }
+            const data = (yield api_insert("register-experiment", args))[0];
+            // NOTE: This is a deviation from the Python lib and allows the log() method
+            // to not be async.
+            //
+            const user_id = (yield _user_info())["id"];
+            return new Experiment(project, data.id, data.name, user_id);
+        });
     }
     /**
      * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
@@ -58,9 +568,38 @@ class Experiment {
      * @returns The `id` of the logged event.
      */
     log({ inputs, output, expected, scores, metadata, }) {
-        // TODO
-        (() => ({ inputs, output, expected, scores, metadata }))();
-        return "foo";
+        for (const [name, score] of Object.entries(scores)) {
+            if (typeof name !== "string") {
+                throw new Error("score names must be strings");
+            }
+            if (typeof score !== "number") {
+                throw new Error("score values must be numbers");
+            }
+            if (score < 0 || score > 1) {
+                throw new Error("score values must be between 0 and 1");
+            }
+        }
+        if (metadata !== undefined) {
+            for (const key of Object.keys(metadata)) {
+                if (typeof key !== "string") {
+                    throw new Error("metadata keys must be strings");
+                }
+            }
+        }
+        const args = {
+            id: (0, uuid_1.v4)(),
+            inputs,
+            output,
+            expected,
+            scores,
+            project_id: this.project.id,
+            experiment_id: this.id,
+            user_id: this.user_id,
+            created: new Date().toISOString(),
+            metadata,
+        };
+        this.logger.log([args]);
+        return args.id;
     }
     /**
      * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
@@ -71,7 +610,41 @@ class Experiment {
      * @returns `ExperimentSummary`
      */
     summarize(options = undefined) {
-        return "Summary!";
+        return __awaiter(this, void 0, void 0, function* () {
+            let { summarizeScores = true, comparisonExperimentId = undefined } = options || {};
+            yield this.logger.flush();
+            const projectUrl = `${API_URL}/app/${encodeURIComponent(ORG_NAME)}/p/${encodeURIComponent(this.project.name)}`;
+            const experimentUrl = `${projectUrl}/${encodeURIComponent(this.name)}`;
+            let scores = undefined;
+            let comparisonExperimentName = undefined;
+            if (summarizeScores) {
+                if (comparisonExperimentId === undefined) {
+                    const conn = api_conn();
+                    const resp = yield conn.get("/crud/base_experiments", {
+                        id: this.id,
+                    });
+                    const base_experiments = resp.data;
+                    if (base_experiments.length > 0) {
+                        comparisonExperimentId = base_experiments[0]["base_exp_id"];
+                        comparisonExperimentName = base_experiments[0]["base_exp_name"];
+                    }
+                }
+                if (comparisonExperimentId !== undefined) {
+                    scores = (yield api_conn().get("/experiment-comparison", {
+                        experiment_id: this.id,
+                        base_experiment_id: comparisonExperimentId,
+                    })).data;
+                }
+            }
+            return {
+                projectName: this.project.name,
+                experimentName: this.name,
+                projectUrl: projectUrl,
+                experimentUrl: experimentUrl,
+                comparisonExperimentName: comparisonExperimentName,
+                scores,
+            };
+        });
     }
 }
 exports.Experiment = Experiment;

package/dist/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"program":{"fileNames":["../node_modules/typescript/lib/lib.es5.d.ts","../node_modules/typescript/lib/lib.es2015.d.ts","../node_modules/typescript/lib/lib.es2015.core.d.ts","../node_modules/typescript/lib/lib.es2015.collection.d.ts","../node_modules/typescript/lib/lib.es2015.generator.d.ts","../node_modules/typescript/lib/lib.es2015.iterable.d.ts","../node_modules/typescript/lib/lib.es2015.promise.d.ts","../node_modules/typescript/lib/lib.es2015.proxy.d.ts","../node_modules/typescript/lib/lib.es2015.reflect.d.ts","../node_modules/typescript/lib/lib.es2015.symbol.d.ts","../node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts","../node_modules/typescript/lib/lib.decorators.d.ts","../node_modules/typescript/lib/lib.decorators.legacy.d.ts","../src/index.ts"],"fileInfos":[{"version":"f59215c5f1d886b05395ee7aca73e0ac69ddfad2843aa88530e797879d511bad","affectsGlobalScope":true},"45b7ab580deca34ae9729e97c13cfd999df04416a79116c3bfb483804f85ded4",{"version":"9d9885c728913c1d16e0d2831b40341d6ad9a0ceecaabc55209b306ad9c736a5","affectsGlobalScope":true},{"version":"17bea081b9c0541f39dd1ae9bc8c78bdd561879a682e60e2f25f688c0ecab248","affectsGlobalScope":true},{"version":"4443e68b35f3332f753eacc66a04ac1d2053b8b035a0e0ac1d455392b5e243b3","affectsGlobalScope":true},{"version":"ab22100fdd0d24cfc2cc59d0a00fc8cf449830d9c4030dc54390a46bd562e929","affectsGlobalScope":true},{"version":"f7bd636ae3a4623c503359ada74510c4005df5b36de7f23e1db8a5c543fd176b","affectsGlobalScope":true},{"version":"ce691fb9e5c64efb9547083e4a34091bcbe5bdb41027e310ebba8f7d96a98671","affectsGlobalScope":true},{"version":"8d697a2a929a5fcb38b7a65594020fcef05ec1630804a33748829c5ff53640d0","affectsGlobalScope":true},{"version":"0c20f4d2358eb679e4ae8a4432bdd96c857a2960fd6800b21ec4008ec59d60ea","affectsGlobalScope":true},{"version":"36ae84ccc0633f7c0787bc6108386c8b773e95d3b052d9464a99cd9b8795fbec","affectsGlobalScope":true},{"version":"189c0703923150aa30673fa3de411346d727cc44a11c75d05d7cf9ef095daa22","affectsGlobalScope":true},{"version":"782dec38049b92d4e85c1585fbea5474a219c6984a35b004963b00beb1aab538","affectsGlobalScope":true},{"version":"0b0efe6cf1c03e6fcdb6e0d4650267739d78670d17e77310d8e4930d91671622","signature":"b4370986352e9c14688509ded0252bf8daa86bdcff5f2c2f7c8b2f7514fec525"}],"root":[14],"options":{"declaration":true,"esModuleInterop":true,"module":1,"outDir":"./","strict":true,"target":2},"referencedMap":[],"exportedModulesMap":[],"semanticDiagnosticsPerFile":[12,13,4,3,2,5,6,7,8,9,10,11,1,14]},"version":"5.1.6"}
+{"program":{"fileNames":["../node_modules/typescript/lib/lib.es5.d.ts","../node_modules/typescript/lib/lib.es2015.d.ts","../node_modules/typescript/lib/lib.es2016.d.ts","../node_modules/typescript/lib/lib.es2017.d.ts","../node_modules/typescript/lib/lib.es2018.d.ts","../node_modules/typescript/lib/lib.es2019.d.ts","../node_modules/typescript/lib/lib.es2020.d.ts","../node_modules/typescript/lib/lib.es2015.core.d.ts","../node_modules/typescript/lib/lib.es2015.collection.d.ts","../node_modules/typescript/lib/lib.es2015.generator.d.ts","../node_modules/typescript/lib/lib.es2015.iterable.d.ts","../node_modules/typescript/lib/lib.es2015.promise.d.ts","../node_modules/typescript/lib/lib.es2015.proxy.d.ts","../node_modules/typescript/lib/lib.es2015.reflect.d.ts","../node_modules/typescript/lib/lib.es2015.symbol.d.ts","../node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts","../node_modules/typescript/lib/lib.es2016.array.include.d.ts","../node_modules/typescript/lib/lib.es2017.object.d.ts","../node_modules/typescript/lib/lib.es2017.sharedmemory.d.ts","../node_modules/typescript/lib/lib.es2017.string.d.ts","../node_modules/typescript/lib/lib.es2017.intl.d.ts","../node_modules/typescript/lib/lib.es2017.typedarrays.d.ts","../node_modules/typescript/lib/lib.es2018.asyncgenerator.d.ts","../node_modules/typescript/lib/lib.es2018.asynciterable.d.ts","../node_modules/typescript/lib/lib.es2018.intl.d.ts","../node_modules/typescript/lib/lib.es2018.promise.d.ts","../node_modules/typescript/lib/lib.es2018.regexp.d.ts","../node_modules/typescript/lib/lib.es2019.array.d.ts","../node_modules/typescript/lib/lib.es2019.object.d.ts","../node_modules/typescript/lib/lib.es2019.string.d.ts","../node_modules/typescript/lib/lib.es2019.symbol.d.ts","../node_modules/typescript/lib/lib.es2019.intl.d.ts","../node_modules/typescript/lib/lib.es2020.bigint.d.ts","../node_modules/typescript/lib/lib.es2020.date.d.ts","../node_modules/typescript/lib/lib.es2020.promise.d.ts","../node_modules/typescript/lib/lib.es2020.sharedmemory.d.ts","../node_modules/typescript/lib/lib.es2020.string.d.ts","../node_modules/typescript/lib/lib.es2020.symbol.wellknown.d.ts","../node_modules/typescript/lib/lib.es2020.intl.d.ts","../node_modules/typescript/lib/lib.es2020.number.d.ts","../node_modules/typescript/lib/lib.esnext.intl.d.ts","../node_modules/typescript/lib/lib.decorators.d.ts","../node_modules/typescript/lib/lib.decorators.legacy.d.ts","../node_modules/axios/index.d.ts","../node_modules/@types/uuid/index.d.ts","../src/index.ts","../node_modules/@types/node/assert.d.ts","../node_modules/@types/node/assert/strict.d.ts","../node_modules/@types/node/globals.d.ts","../node_modules/@types/node/async_hooks.d.ts","../node_modules/@types/node/buffer.d.ts","../node_modules/@types/node/child_process.d.ts","../node_modules/@types/node/cluster.d.ts","../node_modules/@types/node/console.d.ts","../node_modules/@types/node/constants.d.ts","../node_modules/@types/node/crypto.d.ts","../node_modules/@types/node/dgram.d.ts","../node_modules/@types/node/diagnostics_channel.d.ts","../node_modules/@types/node/dns.d.ts","../node_modules/@types/node/dns/promises.d.ts","../node_modules/@types/node/domain.d.ts","../node_modules/@types/node/dom-events.d.ts","../node_modules/@types/node/events.d.ts","../node_modules/@types/node/fs.d.ts","../node_modules/@types/node/fs/promises.d.ts","../node_modules/@types/node/http.d.ts","../node_modules/@types/node/http2.d.ts","../node_modules/@types/node/https.d.ts","../node_modules/@types/node/inspector.d.ts","../node_modules/@types/node/module.d.ts","../node_modules/@types/node/net.d.ts","../node_modules/@types/node/os.d.ts","../node_modules/@types/node/path.d.ts","../node_modules/@types/node/perf_hooks.d.ts","../node_modules/@types/node/process.d.ts","../node_modules/@types/node/punycode.d.ts","../node_modules/@types/node/querystring.d.ts","../node_modules/@types/node/readline.d.ts","../node_modules/@types/node/readline/promises.d.ts","../node_modules/@types/node/repl.d.ts","../node_modules/@types/node/stream.d.ts","../node_modules/@types/node/stream/promises.d.ts","../node_modules/@types/node/stream/consumers.d.ts","../node_modules/@types/node/stream/web.d.ts","../node_modules/@types/node/string_decoder.d.ts","../node_modules/@types/node/test.d.ts","../node_modules/@types/node/timers.d.ts","../node_modules/@types/node/timers/promises.d.ts","../node_modules/@types/node/tls.d.ts","../node_modules/@types/node/trace_events.d.ts","../node_modules/@types/node/tty.d.ts","../node_modules/@types/node/url.d.ts","../node_modules/@types/node/util.d.ts","../node_modules/@types/node/v8.d.ts","../node_modules/@types/node/vm.d.ts","../node_modules/@types/node/wasi.d.ts","../node_modules/@types/node/worker_threads.d.ts","../node_modules/@types/node/zlib.d.ts","../node_modules/@types/node/globals.global.d.ts","../node_modules/@types/node/index.d.ts"],"fileInfos":[{"version":"f59215c5f1d886b05395ee7aca73e0ac69ddfad2843aa88530e797879d511bad","affectsGlobalScope":true},"45b7ab580deca34ae9729e97c13cfd999df04416a79116c3bfb483804f85ded4","dc48272d7c333ccf58034c0026162576b7d50ea0e69c3b9292f803fc20720fd5","27147504487dc1159369da4f4da8a26406364624fa9bc3db632f7d94a5bae2c3","5e1c4c362065a6b95ff952c0eab010f04dcd2c3494e813b493ecfd4fcb9fc0d8","68d73b4a11549f9c0b7d352d10e91e5dca8faa3322bfb77b661839c42b1ddec7","5efce4fc3c29ea84e8928f97adec086e3dc876365e0982cc8479a07954a3efd4",{"version":"9d9885c728913c1d16e0d2831b40341d6ad9a0ceecaabc55209b306ad9c736a5","affectsGlobalScope":true},{"version":"17bea081b9c0541f39dd1ae9bc8c78bdd561879a682e60e2f25f688c0ecab248","affectsGlobalScope":true},{"version":"4443e68b35f3332f753eacc66a04ac1d2053b8b035a0e0ac1d455392b5e243b3","affectsGlobalScope":true},{"version":"ab22100fdd0d24cfc2cc59d0a00fc8cf449830d9c4030dc54390a46bd562e929","affectsGlobalScope":true},{"version":"f7bd636ae3a4623c503359ada74510c4005df5b36de7f23e1db8a5c543fd176b","affectsGlobalScope":true},{"version":"ce691fb9e5c64efb9547083e4a34091bcbe5bdb41027e310ebba8f7d96a98671","affectsGlobalScope":true},{"version":"8d697a2a929a5fcb38b7a65594020fcef05ec1630804a33748829c5ff53640d0","affectsGlobalScope":true},{"version":"0c20f4d2358eb679e4ae8a4432bdd96c857a2960fd6800b21ec4008ec59d60ea","affectsGlobalScope":true},{"version":"36ae84ccc0633f7c0787bc6108386c8b773e95d3b052d9464a99cd9b8795fbec","affectsGlobalScope":true},{"version":"82d0d8e269b9eeac02c3bd1c9e884e85d483fcb2cd168bccd6bc54df663da031","affectsGlobalScope":true},{"version":"b8deab98702588840be73d67f02412a2d45a417a3c097b2e96f7f3a42ac483d1","affectsGlobalScope":true},{"version":"4738f2420687fd85629c9efb470793bb753709c2379e5f85bc1815d875ceadcd","affectsGlobalScope":true},{"version":"2f11ff796926e0832f9ae148008138ad583bd181899ab7dd768a2666700b1893","affectsGlobalScope":true},{"version":"376d554d042fb409cb55b5cbaf0b2b4b7e669619493c5d18d5fa8bd67273f82a","affectsGlobalScope":true},{"version":"9fc46429fbe091ac5ad2608c657201eb68b6f1b8341bd6d670047d32ed0a88fa","affectsGlobalScope":true},{"version":"61c37c1de663cf4171e1192466e52c7a382afa58da01b1dc75058f032ddf0839","affectsGlobalScope":true},{"version":"c4138a3dd7cd6cf1f363ca0f905554e8d81b45844feea17786cdf1626cb8ea06","affectsGlobalScope":true},{"version":"6ff3e2452b055d8f0ec026511c6582b55d935675af67cdb67dd1dc671e8065df","affectsGlobalScope":true},{"version":"03de17b810f426a2f47396b0b99b53a82c1b60e9cba7a7edda47f9bb077882f4","affectsGlobalScope":true},{"version":"8184c6ddf48f0c98429326b428478ecc6143c27f79b79e85740f17e6feb090f1","affectsGlobalScope":true},{"version":"261c4d2cf86ac5a89ad3fb3fafed74cbb6f2f7c1d139b0540933df567d64a6ca","affectsGlobalScope":true},{"version":"6af1425e9973f4924fca986636ac19a0cf9909a7e0d9d3009c349e6244e957b6","affectsGlobalScope":true},{"version":"576711e016cf4f1804676043e6a0a5414252560eb57de9faceee34d79798c850","affectsGlobalScope":true},{"version":"89c1b1281ba7b8a96efc676b11b264de7a8374c5ea1e6617f11880a13fc56dc6","affectsGlobalScope":true},{"version":"15a630d6817718a2ddd7088c4f83e4673fde19fa992d2eae2cf51132a302a5d3","affectsGlobalScope":true},{"version":"f06948deb2a51aae25184561c9640fb66afeddb34531a9212d011792b1d19e0a","affectsGlobalScope":true},{"version":"01e0ee7e1f661acedb08b51f8a9b7d7f959e9cdb6441360f06522cc3aea1bf2e","affectsGlobalScope":true},{"version":"ac17a97f816d53d9dd79b0d235e1c0ed54a8cc6a0677e9a3d61efb480b2a3e4e","affectsGlobalScope":true},{"version":"bf14a426dbbf1022d11bd08d6b8e709a2e9d246f0c6c1032f3b2edb9a902adbe","affectsGlobalScope":true},{"version":"ec0104fee478075cb5171e5f4e3f23add8e02d845ae0165bfa3f1099241fa2aa","affectsGlobalScope":true},{"version":"2b72d528b2e2fe3c57889ca7baef5e13a56c957b946906d03767c642f386bbc3","affectsGlobalScope":true},{"version":"9cc66b0513ad41cb5f5372cca86ef83a0d37d1c1017580b7dace3ea5661836df","affectsGlobalScope":true},{"version":"368af93f74c9c932edd84c58883e736c9e3d53cec1fe24c0b0ff451f529ceab1","affectsGlobalScope":true},{"version":"307c8b7ebbd7f23a92b73a4c6c0a697beca05b06b036c23a34553e5fe65e4fdc","affectsGlobalScope":true},{"version":"189c0703923150aa30673fa3de411346d727cc44a11c75d05d7cf9ef095daa22","affectsGlobalScope":true},{"version":"782dec38049b92d4e85c1585fbea5474a219c6984a35b004963b00beb1aab538","affectsGlobalScope":true},"1d729ea435a93e1a70519d06a6f13fa418c4c39a52b69e6db86750ebfcdf5554","95c617b16c4765ff6de307629b6918181908bee82a59fca0b2c95f170a4be7ea",{"version":"953cc0f7f15219f652f865083404e93e79314a8dedbedfd017320871eac2619c","signature":"503e6fc7e1a057c9c51962e32bac1f31827fae2100f413341595c165936ac140"},"587f13f1e8157bd8cec0adda0de4ef558bb8573daa9d518d1e2af38e87ecc91f","a69c09dbea52352f479d3e7ac949fde3d17b195abe90b045d619f747b38d6d1a",{"version":"bce910d9164785c9f0d4dcea4be359f5f92130c7c7833dea6138ab1db310a1f9","affectsGlobalScope":true},"7a435e0c814f58f23e9a0979045ec0ef5909aac95a70986e8bcce30c27dff228",{"version":"c81c51f43e343b6d89114b17341fb9d381c4ccbb25e0ee77532376052c801ba7","affectsGlobalScope":true},"db71be322f07f769200108aa19b79a75dd19a187c9dca2a30c4537b233aa2863","57135ce61976a8b1dadd01bb412406d1805b90db6e8ecb726d0d78e0b5f76050",{"version":"49479e21a040c0177d1b1bc05a124c0383df7a08a0726ad4d9457619642e875a","affectsGlobalScope":true},"82408ed3e959ddc60d3e9904481b5a8dc16469928257af22a3f7d1a3bc7fd8c4","f302f3a47d7758f67f2afc753b9375d6504dde05d2e6ecdb1df50abbb131fc89","3690133deae19c8127c5505fcb67b04bdc9eb053796008538a9b9abbb70d85aa","5b1c0a23f464f894e7c2b2b6c56df7b9afa60ed48c5345f8618d389a636b2108","be2b092f2765222757c6441b86c53a5ea8dfed47bbc43eab4c5fe37942c866b3","8e6b05abc98adba15e1ac78e137c64576c74002e301d682e66feb77a23907ab8","1ca735bb3d407b2af4fbee7665f3a0a83be52168c728cc209755060ba7ed67bd",{"version":"6b526a5ec4a401ca7c26cfe6a48e641d8f30af76673bad3b06a1b4504594a960","affectsGlobalScope":true},{"version":"b85c02e14ecb2a873dad5a1de72319b265160ba48f1b83661aeb3bba1366c1bc","affectsGlobalScope":true},"7a2ba0c9af860ac3e77b35ed01fd96d15986f17aa22fe40f188ae556fb1070df","fc3764040518a1008dd04bdc80964591b566b896283e00df85c95851c1f46237","55709608060f77965c270ac10ac646286589f1bd1cb174fff1778a2dd9a7ef31","790623a47c5eda62910098884ecb154dc0e5f3a23fc36c1bfb3b5b9ed44e2c2d","42b40e40f2a358cda332456214fad311e1806a6abf3cebaaac72496e07556642","354612fe1d49ecc9551ea3a27d94eef2887b64ef4a71f72ca444efe0f2f0ba80",{"version":"ac0c77cd7db52b3c278bdd1452ce754014835493d05b84535f46854fdc2063b2","affectsGlobalScope":true},"f5490f53d40291cc8607f5463434d1ac6c5564bc4fbb03abceb03a8f6b014457","5e2b91328a540a0933ab5c2203f4358918e6f0fe7505d22840a891a6117735f1","3abc3512fa04aa0230f59ea1019311fd8667bd935d28306311dccc8b17e79d5d",{"version":"14a50dafe3f45713f7f27cb6320dff07c6ac31678f07959c2134260061bf91ff","affectsGlobalScope":true},{"version":"19da7150ca062323b1db6311a6ef058c9b0a39cc64d836b5e9b75d301869653b","affectsGlobalScope":true},"1349077576abb41f0e9c78ec30762ff75b710208aff77f5fdcc6a8c8ce6289dd","e2ce82603102b5c0563f59fb40314cc1ff95a4d521a66ad14146e130ea80d89c","a3e0395220255a350aa9c6d56f882bfcb5b85c19fddf5419ec822cf22246a26d","c27b01e8ddff5cd280711af5e13aecd9a3228d1c256ea797dd64f8fdec5f7df5","898840e876dfd21843db9f2aa6ae38ba2eab550eb780ff62b894b9fbfebfae6b","0cab4d7d4edc40cd3af9eea7c3ed6d1016910c0954c49c4297e479bf3822a625","1b952304137851e45bc009785de89ada562d9376177c97e37702e39e60c2f1ff","785e5be57d4f20f290a20e7b0c6263f6c57fd6e51283050756cef07d6d651c68","44b8b584a338b190a59f4f6929d072431950c7bd92ec2694821c11bce180c8a5","164deb2409ac5f4da3cd139dbcee7f7d66753d90363a4d7e2db8d8874f272270",{"version":"ffc62d73b4fa10ca8c59f8802df88efefe447025730a24ee977b60adedc5bf37","affectsGlobalScope":true},{"version":"ab294c4b7279318ee2a8fdf681305457ecc05970c94108d304933f18823eeac1","affectsGlobalScope":true},"ad08154d9602429522cac965a715fde27d421d69b24756c5d291877dda75353e","5bc85813bfcb6907cc3a960fec8734a29d7884e0e372515147720c5991b8bc22","812b25f798033c202baedf386a1ccc41f9191b122f089bffd10fdccce99fba11","993325544790073f77e945bee046d53988c0bc3ac5695c9cf8098166feb82661",{"version":"75dd741ca6a6c8d2437a6ca8349b64b816421dbf9fe82dd026afaba965576962","affectsGlobalScope":true},{"version":"0e08c360c9b5961ecb0537b703e253842b3ded53151ee07024148219b61a8baf","affectsGlobalScope":true},"2ce2210032ccaff7710e2abf6a722e62c54960458e73e356b6a365c93ab6ca66","92db194ef7d208d5e4b6242a3434573fd142a621ff996d84cc9dbba3553277d0","16a3080e885ed52d4017c902227a8d0d8daf723d062bec9e45627c6fdcd6699b",{"version":"0bd9543cd8fc0959c76fb8f4f5a26626c2ed62ef4be98fd857bce268066db0a2","affectsGlobalScope":true},"1ca6858a0cbcd74d7db72d7b14c5360a928d1d16748a55ecfa6bfaff8b83071b",{"version":"ab9b9a36e5284fd8d3bf2f7d5fcbc60052f25f27e4d20954782099282c60d23e","affectsGlobalScope":true},"ebf3434b09c527078aa74139ff367fffa64fea32a01d6c06fb0a69b0ecadf43e"],"root":[46],"options":{"declaration":true,"esModuleInterop":true,"module":1,"outDir":"./","strict":true,"target":2},"fileIdsList":[[47,93],[50,93],[51,56,84,93],[52,63,64,71,81,92,93],[52,53,63,71,93],[54,93],[55,56,64,72,93],[56,81,89,93],[57,59,63,71,93],[58,93],[59,60,93],[63,93],[61,63,93],[63,64,65,81,92,93],[63,64,65,78,81,84,93],[93,97],[93],[59,63,66,71,81,92,93],[63,64,66,67,71,81,89,92,93],[66,68,81,89,92,93],[47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],[63,69,93],[70,92,93],[59,63,71,81,93],[72,93],[73,93],[50,74,93],[75,91,93,97],[76,93],[77,93],[63,78,79,93],[78,80,93,95],[51,63,81,82,83,84,93],[51,81,83,93],[81,82,93],[84,93],[85,93],[81,93],[63,87,88,93],[87,88,93],[56,71,81,89,93],[90,93],[71,91,93],[51,66,77,92,93],[56,93],[81,93,94],[93,95],[93,96],[51,56,63,65,74,81,92,93,95,97],[81,93,98],[44,45,66,68,93]],"referencedMap":[[47,1],[48,1],[50,2],[51,3],[52,4],[53,5],[54,6],[55,7],[56,8],[57,9],[58,10],[59,11],[60,11],[62,12],[61,13],[63,12],[64,14],[65,15],[49,16],[99,17],[66,18],[67,19],[68,20],[100,21],[69,22],[70,23],[71,24],[72,25],[73,26],[74,27],[75,28],[76,29],[77,30],[78,31],[79,31],[80,32],[81,33],[83,34],[82,35],[84,36],[85,37],[86,38],[87,39],[88,40],[89,41],[90,42],[91,43],[92,44],[93,45],[94,46],[95,47],[96,48],[97,49],[98,50],[45,17],[44,17],[42,17],[43,17],[9,17],[8,17],[2,17],[10,17],[11,17],[12,17],[13,17],[14,17],[15,17],[16,17],[17,17],[3,17],[4,17],[21,17],[18,17],[19,17],[20,17],[22,17],[23,17],[24,17],[5,17],[25,17],[26,17],[27,17],[28,17],[6,17],[32,17],[29,17],[30,17],[31,17],[33,17],[7,17],[34,17],[39,17],[40,17],[35,17],[36,17],[37,17],[38,17],[1,17],[41,17],[46,51]],"exportedModulesMap":[[47,1],[48,1],[50,2],[51,3],[52,4],[53,5],[54,6],[55,7],[56,8],[57,9],[58,10],[59,11],[60,11],[62,12],[61,13],[63,12],[64,14],[65,15],[49,16],[99,17],[66,18],[67,19],[68,20],[100,21],[69,22],[70,23],[71,24],[72,25],[73,26],[74,27],[75,28],[76,29],[77,30],[78,31],[79,31],[80,32],[81,33],[83,34],[82,35],[84,36],[85,37],[86,38],[87,39],[88,40],[89,41],[90,42],[91,43],[92,44],[93,45],[94,46],[95,47],[96,48],[97,49],[98,50],[45,17],[44,17],[17,17],[3,17],[4,17],[21,17],[18,17],[19,17],[20,17],[22,17],[23,17],[24,17],[5,17],[25,17],[26,17],[27,17],[28,17],[6,17],[32,17],[29,17],[30,17],[31,17],[33,17],[7,17],[34,17],[39,17],[40,17],[35,17],[36,17],[37,17],[38,17],[41,17]],"semanticDiagnosticsPerFile":[47,48,50,51,52,53,54,55,56,57,58,59,60,62,61,63,64,65,49,99,66,67,68,100,69,70,71,72,73,74,75,76,77,78,79,80,81,83,82,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,45,44,42,43,9,8,2,10,11,12,13,14,15,16,17,3,4,21,18,19,20,22,23,24,5,25,26,27,28,6,32,29,30,31,33,7,34,39,40,35,36,37,38,1,41,46]},"version":"5.1.6"}

package/package.json

@@ -1,15 +1,26 @@
 {
   "name": "braintrust",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "SDK for integrating BrainTrust",
   "main": "dist/index.js",
   "scripts": {
     "build": "tsc",
+    "watch": "tsc --watch",
+    "docs": "npx typedoc --plugin typedoc-plugin-markdown --out ../app/pages/docs/libs/js src/index.ts --publicPath /docs/libs/js/ --githubPages false --disableSources true --hideInPageTOC true",
     "test": "jest"
   },
   "author": "",
   "license": "MIT",
   "devDependencies": {
+    "@types/uuid": "^9.0.2",
+    "typedoc": "^0.24.8",
+    "typedoc-plugin-markdown": "^3.15.3",
     "typescript": "^5.1.6"
+  },
+  "dependencies": {
+    "@types/axios": "^0.14.0",
+    "@types/node": "^20.4.1",
+    "axios": "^1.4.0",
+    "uuid": "^9.0.0"
   }
 }