braintrust 0.0.21 → 0.0.23

package/dist/jest/nodeModulesPaths.js

@@ -0,0 +1,101 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * Adapted from: https://github.com/substack/node-resolve
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GlobalPaths = void 0;
+const path = __importStar(require("path"));
+// BRAINTRUST: This was changed to be a relative import
+const tryRealpath_1 = __importDefault(require("./tryRealpath"));
+function nodeModulesPaths(basedir, options) {
+    const modules = options && options.moduleDirectory
+        ? Array.from(options.moduleDirectory)
+        : ["node_modules"];
+    // ensure that `basedir` is an absolute path at this point,
+    // resolving against the process' current working directory
+    const basedirAbs = path.resolve(basedir);
+    let prefix = "/";
+    if (/^([A-Za-z]:)/.test(basedirAbs)) {
+        prefix = "";
+    }
+    else if (/^\\\\/.test(basedirAbs)) {
+        prefix = "\\\\";
+    }
+    // The node resolution algorithm (as implemented by NodeJS and TypeScript)
+    // traverses parents of the physical path, not the symlinked path
+    let physicalBasedir;
+    try {
+        physicalBasedir = (0, tryRealpath_1.default)(basedirAbs);
+    }
+    catch (_a) {
+        // realpath can throw, e.g. on mapped drives
+        physicalBasedir = basedirAbs;
+    }
+    const paths = [physicalBasedir];
+    let parsed = path.parse(physicalBasedir);
+    while (parsed.dir !== paths[paths.length - 1]) {
+        paths.push(parsed.dir);
+        parsed = path.parse(parsed.dir);
+    }
+    const dirs = paths.reduce((dirs, aPath) => {
+        for (const moduleDir of modules) {
+            if (path.isAbsolute(moduleDir)) {
+                if (aPath === basedirAbs && moduleDir) {
+                    dirs.push(moduleDir);
+                }
+            }
+            else {
+                dirs.push(path.join(prefix, aPath, moduleDir));
+            }
+        }
+        return dirs;
+    }, []);
+    if (options.paths) {
+        dirs.push(...options.paths);
+    }
+    return dirs;
+}
+exports.default = nodeModulesPaths;
+function findGlobalPaths() {
+    const { root } = path.parse(process.cwd());
+    const globalPath = path.join(root, "node_modules");
+    const resolvePaths = require.resolve.paths("/");
+    if (resolvePaths) {
+        // the global paths start one after the root node_modules
+        const rootIndex = resolvePaths.indexOf(globalPath);
+        return rootIndex > -1 ? resolvePaths.slice(rootIndex + 1) : [];
+    }
+    return [];
+}
+exports.GlobalPaths = findGlobalPaths();

package/dist/jest/tryRealpath.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+export default function tryRealpath(path: string): string;

package/dist/jest/tryRealpath.js

@@ -0,0 +1,21 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const graceful_fs_1 = require("graceful-fs");
+function tryRealpath(path) {
+    try {
+        path = graceful_fs_1.realpathSync.native(path);
+    }
+    catch (error) {
+        if (error.code !== "ENOENT" && error.code !== "EISDIR") {
+            throw error;
+        }
+    }
+    return path;
+}
+exports.default = tryRealpath;

package/dist/logger.d.ts

@@ -0,0 +1,206 @@
+export declare class Project {
+    name: string;
+    id: string;
+    org_id: string;
+    constructor(name: string, id: string, org_id: string);
+}
+/**
+ * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
+ *
+ * @param project The name of the project to create the experiment in.
+ * @param options Additional options for configuring init().
+ * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
+ * @param options.description An optional description of the experiment.
+ * @param options.update If the experiment already exists, continue logging to it.
+ * @param options.baseExperiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
+ * experiment. Otherwise, it will pick an experiment by finding the closest ancestor on the default (e.g. main) branch.
+ * @param options.apiUrl The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disableCache Do not use cached login information.
+ * @returns The newly created Experiment.
+ */
+export declare function init(project: string, options?: {
+    readonly experiment?: string;
+    readonly description?: string;
+    readonly update?: boolean;
+    readonly baseExperiment?: string;
+    readonly apiUrl?: string;
+    readonly apiKey?: string;
+    readonly orgName?: string;
+    readonly disableCache?: boolean;
+}): Promise<Experiment>;
+/**
+ * Log into BrainTrust. This will prompt you for your API token, which you can find at
+ * https://www.braintrustdata.com/app/token. This method is called automatically by `init()`.
+ *
+ * @param options Options for configuring login().
+ * @param options.apiUrl The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disableCache Do not use cached login information.
+ * @param options.forceLogin Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
+ */
+export declare function login(options?: {
+    apiUrl?: string;
+    apiKey?: string;
+    orgName?: string;
+    disableCache?: boolean;
+    forceLogin?: boolean;
+}): Promise<void>;
+/**
+ * Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
+ *
+ * @param event The event to log.
+ * @param event.input The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+ * BrainTrust will use the `input` to know whether two test casess are the same between experiments, so they should
+ * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+ * `input` should be identical.
+ * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+ * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+ * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+ * be multiple valid queries that answer a single question.
+ * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+ * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+ * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+ * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+ * fine-tune your models.
+ * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+ * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+ * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+ * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+ * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+ * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+ * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+ * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+ * JSON-serializable type, but its keys must be strings.
+ * @param event.id (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
+ * @param event.inputs (Deprecated) the same as `input` (will be removed in a future version)
+ * @returns The `id` of the logged event.
+ */
+export declare function log(options: {
+    readonly input?: unknown;
+    readonly output: unknown;
+    readonly expected?: unknown;
+    readonly scores: Record<string, number>;
+    readonly metadata?: Record<string, unknown>;
+    readonly id?: string;
+    readonly inputs?: unknown;
+}): string;
+/**
+ * Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
+ *
+ * @param options Options for summarizing the experiment.
+ * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
+ * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+ * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
+ */
+export declare function summarize(options?: {
+    readonly summarizeScores?: boolean;
+    readonly comparisonExperimentId?: string;
+}): Promise<ExperimentSummary>;
+/**
+ * An experiment is a collection of logged events, such as model inputs and outputs, which represent
+ * a snapshot of your application at a particular point in time. An experiment is meant to capture more
+ * than just the model you use, and includes the data you use to test, pre- and post- processing code,
+ * comparison metrics (scores), and any other metadata you want to include.
+ *
+ * Experiments are associated with a project, and two experiments are meant to be easily comparable via
+ * their `inputs`. You can change the attributes of the experiments in a project (e.g. scoring functions)
+ * over time, simply by changing what you log.
+ *
+ * You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
+ */
+export declare class Experiment {
+    readonly project: Project;
+    readonly id: string;
+    readonly name: string;
+    readonly user_id: string;
+    private logger;
+    constructor(project: Project, id: string, name: string, user_id: string);
+    /**
+     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+     *
+     * @param event The event to log.
+     * @param event.input The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * BrainTrust will use the `input` to know whether two test casess are the same between experiments, so they should
+     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+     * `input` should be identical.
+     * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+     * be multiple valid queries that answer a single question.
+     * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+     * fine-tune your models.
+     * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+     * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+     * JSON-serializable type, but its keys must be strings.
+     * @param event.id (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
+     * @param event.inputs (Deprecated) the same as `input` (will be removed in a future version)
+     * @returns The `id` of the logged event.
+     */
+    log({ input, output, expected, scores, metadata, id, inputs, }: {
+        readonly input?: unknown;
+        readonly output: unknown;
+        readonly expected?: unknown;
+        readonly scores: Record<string, number>;
+        readonly metadata?: Record<string, unknown>;
+        readonly id?: string;
+        readonly inputs?: unknown;
+    }): string;
+    /**
+     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     *
+     * @param options Options for summarizing the experiment.
+     * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     */
+    summarize(options?: {
+        readonly summarizeScores?: boolean;
+        readonly comparisonExperimentId?: string;
+    }): Promise<ExperimentSummary>;
+}
+/**
+ * Summary of a score's performance.
+ * @property name Name of the score.
+ * @property score Average score across all examples.
+ * @property diff Difference in score between the current and reference experiment.
+ * @property improvements Number of improvements in the score.
+ * @property regressions Number of regressions in the score.
+ */
+export interface ScoreSummary {
+    name: string;
+    score: number;
+    diff: number;
+    improvements: number;
+    regressions: number;
+}
+/**
+ * Summary of an experiment's scores and metadata.
+ * @property projectName Name of the project that the experiment belongs to.
+ * @property experimentName Name of the experiment.
+ * @property projectUrl URL to the project's page in the BrainTrust app.
+ * @property experimentUrl URL to the experiment's page in the BrainTrust app.
+ * @property comparisonExperimentName The experiment scores are baselined against.
+ * @property scores Summary of the experiment's scores.
+ */
+export interface ExperimentSummary {
+    projectName: string;
+    experimentName: string;
+    projectUrl: string;
+    experimentUrl: string;
+    comparisonExperimentName: string | undefined;
+    scores: Record<string, ScoreSummary> | undefined;
+}