braintrust 0.0.2 → 0.0.3

package/dist/api.js

@@ -0,0 +1 @@
+"use strict";

package/dist/dep.d.ts

@@ -0,0 +1,52 @@
+export declare class Experiment {
+    private _project;
+    constructor(project: string);
+    get project(): string;
+    /**
+     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+     *
+     * @param values
+     * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+     * `inputs` should be identical.
+     * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+     * be multiple valid queries that answer a single question.
+     * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+     * fine-tune your models.
+     * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+     * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+     * JSON-serializable type, but its keys must be strings.
+     * @returns The `id` of the logged event.
+     */
+    log({ inputs, output, expected, scores, metadata, }: {
+        readonly inputs: unknown;
+        readonly output: unknown;
+        readonly expected: unknown;
+        readonly scores: Record<string, number>;
+        readonly metadata?: Record<string, unknown>;
+    }): string;
+    /**
+     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     *
+     * @param options
+     * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns `ExperimentSummary`
+     */
+    summarize(options?: {
+        readonly summarizeScores?: boolean;
+        readonly comparisonExperimentId?: string;
+    } | undefined): string;
+}

package/dist/dep.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Experiment = void 0;
+class Experiment {
+    constructor(project) {
+        this._project = project;
+    }
+    get project() {
+        return this._project;
+    }
+    /**
+     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+     *
+     * @param values
+     * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+     * `inputs` should be identical.
+     * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+     * be multiple valid queries that answer a single question.
+     * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+     * fine-tune your models.
+     * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+     * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+     * JSON-serializable type, but its keys must be strings.
+     * @returns The `id` of the logged event.
+     */
+    log({ inputs, output, expected, scores, metadata, }) {
+        // TODO
+        (() => ({ inputs, output, expected, scores, metadata }))();
+        return "foo";
+    }
+    /**
+     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     *
+     * @param options
+     * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns `ExperimentSummary`
+     */
+    summarize(options = undefined) {
+        return "Summary!";
+    }
+}
+exports.Experiment = Experiment;

package/dist/index.d.ts

@@ -1,8 +1,48 @@
+/**
+ * A Node.js library for logging data to BrainTrust.
+ *
+ * ### Quickstart
+ *
+ * Install the library with npm (or yarn).
+ *
+ * ```bash
+ * npm install braintrust
+ * ```
+ *
+ * Then, run a simple experiment with the following code (replace `YOUR_API_KEY` with
+ * your BrainTrust API key):
+ *
+ * ```javascript
+ * const braintrust = require("braintrust");
+ *
+ * const experiment = await braintrust.init("NodeTest", {api_key: "YOUR_API_KEY"});
+ * experiment.log({
+ *   inputs: {test: 1},
+ *   output: "foo",
+ *   expected: "bar",
+ *   scores: {
+ *     n: 0.5,
+ *   },
+ *   metadata: {
+ *     id: 1,
+ *   },
+ * });
+ * console.log(await experiment.summarize());
+ * ```
+ *
+ * @module braintrust
+ */
+export declare class Project {
+    name: string;
+    id: string;
+    org_id: string;
+    constructor(name: string, id: string, org_id: string);
+}
 /**
  * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
  *
  * @param project The name of the project to create the experiment in.
- * @param options
+ * @param options Additional options for configuring init().
  * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
  * @param options.description An optional description of the experiment.
  * @param options.base_experiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
@@ -14,7 +54,7 @@
  * @param options.disable_cache Do not use cached login information.
  * @returns The experiment object.
  */
-export declare function init(project: string, { experiment, }: {
+export declare function init(project: string, options?: {
     readonly experiment?: string;
     readonly description?: string;
     readonly base_experiment?: string;
@@ -22,11 +62,98 @@ export declare function init(project: string, { experiment, }: {
     readonly api_key?: string;
     readonly org_name?: string;
     readonly disable_cache?: boolean;
-}): Experiment;
+}): Promise<Experiment>;
+/**
+ * Log into BrainTrust. This will prompt you for your API token, which you can find at
+ * https://www.braintrustdata.com/app/token. This method is called automatically by `init()`.
+ *
+ * @param options
+ * @param options.api_url The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.api_key The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.org_name (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disable_cache Do not use cached login information.
+ * @param options.force_login Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
+ * @returns
+ */
+export declare function login(options?: {
+    api_url?: string;
+    api_key?: string;
+    org_name?: string;
+    disable_cache?: boolean;
+    force_login?: boolean;
+} | undefined): Promise<void>;
+/**
+ * Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
+ *
+ * @param values
+ * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+ * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+ * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+ * `inputs` should be identical.
+ * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+ * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+ * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+ * be multiple valid queries that answer a single question.
+ * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+ * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+ * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+ * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+ * fine-tune your models.
+ * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+ * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+ * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+ * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+ * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+ * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+ * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+ * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+ * JSON-serializable type, but its keys must be strings.
+ * @returns The `id` of the logged event.
+ */
+export declare function log(options: {
+    readonly inputs: unknown;
+    readonly output: unknown;
+    readonly expected: unknown;
+    readonly scores: Record<string, number>;
+    readonly metadata?: Record<string, unknown>;
+}): string;
+/**
+ * Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
+ *
+ * @param options
+ * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+ * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+ * @returns `ExperimentSummary`
+ */
+export declare function summarize(options?: {
+    readonly summarizeScores?: boolean;
+    readonly comparisonExperimentId?: string;
+} | undefined): Promise<ExperimentSummary>;
+/**
+ * An experiment is a collection of logged events, such as model inputs and outputs, which represent
+ * a snapshot of your application at a particular point in time. An experiment is meant to capture more
+ * than just the model you use, and includes the data you use to test, pre- and post- processing code,
+ * comparison metrics (scores), and any other metadata you want to include.
+ *
+ * Experiments are associated with a project, and two experiments are meant to be easily comparable via
+ * their `inputs`. You can change the attributes of the experiments in a project (e.g. scoring functions)
+ * over time, simply by changing what you log.
+ *
+ * You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
+ */
 export declare class Experiment {
-    private _project;
-    constructor(project: string);
-    get project(): string;
+    readonly project: Project;
+    readonly id: string;
+    readonly name: string;
+    readonly user_id: string;
+    private logger;
+    constructor(project: Project, id: string, name: string, user_id: string);
+    static init(project: Project, { name, description, base_experiment, }?: {
+        name?: string;
+        description?: string;
+        base_experiment?: string;
+    }): Promise<Experiment>;
     /**
      * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
      *
@@ -73,5 +200,38 @@ export declare class Experiment {
     summarize(options?: {
         readonly summarizeScores?: boolean;
         readonly comparisonExperimentId?: string;
-    } | undefined): string;
+    } | undefined): Promise<ExperimentSummary>;
+}
+/**
+ * Summary of a score's performance.
+ * @property name Name of the score.
+ * @property score Average score across all examples.
+ * @property diff Difference in score between the current and reference experiment.
+ * @property improvements Number of improvements in the score.
+ * @property regressions Number of regressions in the score.
+ */
+interface ScoreSummary {
+    name: string;
+    score: number;
+    diff: number;
+    improvements: number;
+    regressions: number;
+}
+/**
+ * Summary of an experiment's scores and metadata.
+ * @property projectName Name of the project that the experiment belongs to.
+ * @property experimentName Name of the experiment.
+ * @property projectUrl URL to the project's page in the BrainTrust app.
+ * @property experimentUrl URL to the experiment's page in the BrainTrust app.
+ * @property comparisonExperimentName The experiment scores are baselined against.
+ * @property scores Summary of the experiment's scores.
+ */
+interface ExperimentSummary {
+    projectName: string;
+    experimentName: string;
+    projectUrl: string;
+    experimentUrl: string;
+    comparisonExperimentName: string | undefined;
+    scores: Record<string, ScoreSummary> | undefined;
 }
+export {};