braintrust 0.0.3 → 0.0.11

package/README.md

@@ -0,0 +1,32 @@
+A Node.js library for logging data to BrainTrust.
+
+### Quickstart
+
+Install the library with npm (or yarn).
+
+```bash
+npm install braintrust
+```
+
+Then, run a simple experiment with the following code (replace `YOUR_API_KEY` with
+your BrainTrust API key):
+
+```javascript
+const braintrust = require("braintrust");
+
+const experiment = await braintrust.init("NodeTest", {
+  api_key: "YOUR_API_KEY",
+});
+experiment.log({
+  inputs: { test: 1 },
+  output: "foo",
+  expected: "bar",
+  scores: {
+    n: 0.5,
+  },
+  metadata: {
+    id: 1,
+  },
+});
+console.log(await experiment.summarize());
+```

package/dist/cache.d.ts

@@ -0,0 +1,3 @@
+export declare const CACHE_PATH: string;
+export declare const EXPERIMENTS_PATH: string;
+export declare const LOGIN_INFO_PATH: string;

package/dist/cache.js

@@ -0,0 +1,31 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LOGIN_INFO_PATH = exports.EXPERIMENTS_PATH = exports.CACHE_PATH = void 0;
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+exports.CACHE_PATH = path.join(os.homedir(), ".cache", "braintrust");
+exports.EXPERIMENTS_PATH = path.join(exports.CACHE_PATH, "experiments");
+exports.LOGIN_INFO_PATH = path.join(exports.CACHE_PATH, "api_info.json");

package/dist/gitutil.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Information about the current HEAD of the repo.
+ */
+export interface RepoStatus {
+    commit?: string;
+    branch?: string;
+    tag?: string;
+    dirty: boolean;
+    author_name?: string;
+    author_email?: string;
+    commit_message?: string;
+    commit_time?: string;
+}
+export declare function currentRepo(): Promise<import("simple-git").SimpleGit | null>;
+export declare function getPastNAncestors(n?: number, remote?: string | undefined): Promise<string[]>;
+export declare function getRepoStatus(): Promise<{
+    commit: string | undefined;
+    branch: string | undefined;
+    tag: string | undefined;
+    dirty: boolean;
+    author_name: string | undefined;
+    author_email: string | undefined;
+    commit_message: string | undefined;
+    commit_time: string | undefined;
+} | undefined>;

package/dist/gitutil.js

@@ -0,0 +1,136 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRepoStatus = exports.getPastNAncestors = exports.currentRepo = void 0;
+const simple_git_1 = require("simple-git");
+function currentRepo() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const git = (0, simple_git_1.simpleGit)();
+        if (yield git.checkIsRepo()) {
+            return git;
+        }
+        else {
+            return null;
+        }
+    });
+}
+exports.currentRepo = currentRepo;
+let _baseBranch = null;
+function getBaseBranch(remote = undefined) {
+    var _a;
+    return __awaiter(this, void 0, void 0, function* () {
+        if (_baseBranch === null) {
+            const git = yield currentRepo();
+            if (git === null) {
+                throw new Error("Not in a git repo");
+            }
+            const remoteName = remote !== null && remote !== void 0 ? remote : (_a = (yield git.getRemotes())[0]) === null || _a === void 0 ? void 0 : _a.name;
+            if (!remoteName) {
+                // TODO: We should fix this in the Python SDK too. If you have a repo with no remotes, it will
+                // fail with a cryptic error message.
+                throw new Error("No remote found");
+            }
+            const remoteInfo = yield git.remote(["show", remoteName]);
+            if (!remoteInfo) {
+                throw new Error(`Could not find remote ${remoteName}`);
+            }
+            const match = remoteInfo.match(/\s*HEAD branch:\s*(.*)$/m);
+            if (!match) {
+                throw new Error(`Could not find HEAD branch in remote ${remoteName}`);
+            }
+            _baseBranch = { remote: remoteName, branch: match[1] };
+        }
+        return _baseBranch;
+    });
+}
+function getBaseBranchAncestor(remote = undefined) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const git = yield currentRepo();
+        if (git === null) {
+            throw new Error("Not in a git repo");
+        }
+        const { remote: remoteName, branch: baseBranch } = yield getBaseBranch(remote);
+        const isDirty = (yield git.diffSummary()).files.length > 0;
+        const head = isDirty ? "HEAD" : "HEAD^";
+        try {
+            const ancestor = yield git.raw([
+                "merge-base",
+                head,
+                `${remoteName}/${baseBranch}`,
+            ]);
+            return ancestor.trim();
+        }
+        catch (e) {
+            console.warn(`Could not find a common ancestor with ${remoteName}/${baseBranch}`, e);
+            return undefined;
+        }
+    });
+}
+function getPastNAncestors(n = 10, remote = undefined) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const git = yield currentRepo();
+        if (git === null) {
+            return [];
+        }
+        const ancestor = yield getBaseBranchAncestor(remote);
+        const commits = yield git.log({ from: ancestor, to: "HEAD" });
+        return commits.all.map((c) => c.hash);
+    });
+}
+exports.getPastNAncestors = getPastNAncestors;
+function attempt(fn) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            return yield fn();
+        }
+        catch (e) {
+            return undefined;
+        }
+    });
+}
+function getRepoStatus() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const git = yield currentRepo();
+        if (git === null) {
+            return undefined;
+        }
+        let commit = undefined;
+        let commit_message = undefined;
+        let commit_time = undefined;
+        let author_name = undefined;
+        let author_email = undefined;
+        let tag = undefined;
+        let branch = undefined;
+        const dirty = (yield git.diffSummary()).files.length > 0;
+        if (!dirty) {
+            commit = yield attempt(() => __awaiter(this, void 0, void 0, function* () { return yield git.revparse(["HEAD"]); }));
+            commit_message = yield attempt(() => __awaiter(this, void 0, void 0, function* () { return (yield git.raw(["log", "-1", "--pretty=%B"])).trim(); }));
+            commit_time = yield attempt(() => __awaiter(this, void 0, void 0, function* () { return (yield git.raw(["log", "-1", "--pretty=%cI"])).trim(); }));
+            author_name = yield attempt(() => __awaiter(this, void 0, void 0, function* () { return (yield git.raw(["log", "-1", "--pretty=%aN"])).trim(); }));
+            author_email = yield attempt(() => __awaiter(this, void 0, void 0, function* () { return (yield git.raw(["log", "-1", "--pretty=%aE"])).trim(); }));
+            tag = yield attempt(() => __awaiter(this, void 0, void 0, function* () {
+                return (yield git.raw(["describe", "--tags", "--exact-match", "--always"])).trim();
+            }));
+        }
+        branch = yield attempt(() => __awaiter(this, void 0, void 0, function* () { return (yield git.raw(["rev-parse", "--abbrev-ref", "HEAD"])).trim(); }));
+        return {
+            commit,
+            branch,
+            tag,
+            dirty,
+            author_name,
+            author_email,
+            commit_message,
+            commit_time,
+        };
+    });
+}
+exports.getRepoStatus = getRepoStatus;

package/dist/index.d.ts

@@ -52,7 +52,7 @@ export declare class Project {
  * key is specified, will prompt the user to login.
  * @param options.org_name (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
  * @param options.disable_cache Do not use cached login information.
- * @returns The experiment object.
+ * @returns The newly created Experiment.
  */
 export declare function init(project: string, options?: {
     readonly experiment?: string;
@@ -67,14 +67,13 @@ export declare function init(project: string, options?: {
  * Log into BrainTrust. This will prompt you for your API token, which you can find at
  * https://www.braintrustdata.com/app/token. This method is called automatically by `init()`.
  *
- * @param options
+ * @param options Options for configuring login().
  * @param options.api_url The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
  * @param options.api_key The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
  * key is specified, will prompt the user to login.
  * @param options.org_name (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
  * @param options.disable_cache Do not use cached login information.
  * @param options.force_login Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
- * @returns
  */
 export declare function login(options?: {
     api_url?: string;
@@ -82,30 +81,30 @@ export declare function login(options?: {
     org_name?: string;
     disable_cache?: boolean;
     force_login?: boolean;
-} | undefined): Promise<void>;
+}): Promise<void>;
 /**
  * Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
  *
- * @param values
- * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+ * @param event The event to log.
+ * @param event.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
  * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
  * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
  * `inputs` should be identical.
- * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+ * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
  * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
  * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
  * be multiple valid queries that answer a single question.
- * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+ * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
  * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
  * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
  * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
  * fine-tune your models.
- * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+ * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
  * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
  * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
  * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
  * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
- * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+ * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
  * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
  * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
  * JSON-serializable type, but its keys must be strings.
@@ -121,15 +120,15 @@ export declare function log(options: {
 /**
  * Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
  *
- * @param options
- * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
- * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
- * @returns `ExperimentSummary`
+ * @param options Options for summarizing the experiment.
+ * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
+ * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+ * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
  */
 export declare function summarize(options?: {
     readonly summarizeScores?: boolean;
     readonly comparisonExperimentId?: string;
-} | undefined): Promise<ExperimentSummary>;
+}): Promise<ExperimentSummary>;
 /**
  * An experiment is a collection of logged events, such as model inputs and outputs, which represent
  * a snapshot of your application at a particular point in time. An experiment is meant to capture more
@@ -149,34 +148,29 @@ export declare class Experiment {
     readonly user_id: string;
     private logger;
     constructor(project: Project, id: string, name: string, user_id: string);
-    static init(project: Project, { name, description, base_experiment, }?: {
-        name?: string;
-        description?: string;
-        base_experiment?: string;
-    }): Promise<Experiment>;
     /**
      * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
      *
-     * @param values
-     * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * @param event The event to log.
+     * @param event.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
      * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
      * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
      * `inputs` should be identical.
-     * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
      * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
      * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
      * be multiple valid queries that answer a single question.
-     * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
      * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
      * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
      * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
      * fine-tune your models.
-     * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
      * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
      * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
      * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
      * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
-     * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
      * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
      * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
      * JSON-serializable type, but its keys must be strings.
@@ -192,15 +186,15 @@ export declare class Experiment {
     /**
      * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
      *
-     * @param options
-     * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
-     * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
-     * @returns `ExperimentSummary`
+     * @param options Options for summarizing the experiment.
+     * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
      */
     summarize(options?: {
         readonly summarizeScores?: boolean;
         readonly comparisonExperimentId?: string;
-    } | undefined): Promise<ExperimentSummary>;
+    }): Promise<ExperimentSummary>;
 }
 /**
  * Summary of a score's performance.
@@ -210,7 +204,7 @@ export declare class Experiment {
  * @property improvements Number of improvements in the score.
  * @property regressions Number of regressions in the score.
  */
-interface ScoreSummary {
+export interface ScoreSummary {
     name: string;
     score: number;
     diff: number;
@@ -226,7 +220,7 @@ interface ScoreSummary {
  * @property comparisonExperimentName The experiment scores are baselined against.
  * @property scores Summary of the experiment's scores.
  */
-interface ExperimentSummary {
+export interface ExperimentSummary {
     projectName: string;
     experimentName: string;
     projectUrl: string;
@@ -234,4 +228,3 @@ interface ExperimentSummary {
     comparisonExperimentName: string | undefined;
     scores: Record<string, ScoreSummary> | undefined;
 }
-export {};