braintrust 0.0.1

package/dist/index.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
+ *
+ * @param project The name of the project to create the experiment in.
+ * @param options
+ * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
+ * @param options.description An optional description of the experiment.
+ * @param options.base_experiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
+ * experiment. Otherwise, it will pick an experiment by finding the closest ancestor on the default (e.g. main) branch.
+ * @param options.api_url The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.api_key The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.org_name (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disable_cache Do not use cached login information.
+ * @returns The experiment object.
+ */
+export declare function init(project: string, { experiment, }: {
+    readonly experiment?: string;
+    readonly description?: string;
+    readonly base_experiment?: string;
+    readonly api_url?: string;
+    readonly api_key?: string;
+    readonly org_name?: string;
+    readonly disable_cache?: boolean;
+}): Experiment;
+export declare class Experiment {
+    private _project;
+    constructor(project: string);
+    get project(): string;
+    /**
+     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+     *
+     * @param values
+     * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+     * `inputs` should be identical.
+     * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+     * be multiple valid queries that answer a single question.
+     * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+     * fine-tune your models.
+     * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+     * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+     * JSON-serializable type, but its keys must be strings.
+     * @returns The `id` of the logged event.
+     */
+    log({ inputs, output, expected, scores, metadata, }: {
+        readonly inputs: unknown;
+        readonly output: unknown;
+        readonly expected: unknown;
+        readonly scores: Record<string, number>;
+        readonly metadata?: Record<string, unknown>;
+    }): string;
+    /**
+     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     *
+     * @param options
+     * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns `ExperimentSummary`
+     */
+    summarize(options?: {
+        readonly summarizeScores?: boolean;
+        readonly comparisonExperimentId?: string;
+    } | undefined): string;
+}

package/dist/index.js

@@ -0,0 +1,77 @@
+"use strict";
+/**
+ * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
+ *
+ * @param project The name of the project to create the experiment in.
+ * @param options
+ * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
+ * @param options.description An optional description of the experiment.
+ * @param options.base_experiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
+ * experiment. Otherwise, it will pick an experiment by finding the closest ancestor on the default (e.g. main) branch.
+ * @param options.api_url The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.api_key The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.org_name (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disable_cache Do not use cached login information.
+ * @returns The experiment object.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Experiment = exports.init = void 0;
+function init(project, { experiment, }) {
+    // TODO
+    return new Experiment(project);
+}
+exports.init = init;
+class Experiment {
+    constructor(project) {
+        this._project = project;
+    }
+    get project() {
+        return this._project;
+    }
+    /**
+     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+     *
+     * @param values
+     * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+     * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+     * `inputs` should be identical.
+     * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+     * be multiple valid queries that answer a single question.
+     * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+     * fine-tune your models.
+     * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+     * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+     * JSON-serializable type, but its keys must be strings.
+     * @returns The `id` of the logged event.
+     */
+    log({ inputs, output, expected, scores, metadata, }) {
+        // TODO
+        (() => ({ inputs, output, expected, scores, metadata }))();
+        return "foo";
+    }
+    /**
+     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+     *
+     * @param options
+     * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+     * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+     * @returns `ExperimentSummary`
+     */
+    summarize(options = undefined) {
+        return "Summary!";
+    }
+}
+exports.Experiment = Experiment;

package/dist/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"program":{"fileNames":["../node_modules/typescript/lib/lib.es5.d.ts","../node_modules/typescript/lib/lib.es2015.d.ts","../node_modules/typescript/lib/lib.es2015.core.d.ts","../node_modules/typescript/lib/lib.es2015.collection.d.ts","../node_modules/typescript/lib/lib.es2015.generator.d.ts","../node_modules/typescript/lib/lib.es2015.iterable.d.ts","../node_modules/typescript/lib/lib.es2015.promise.d.ts","../node_modules/typescript/lib/lib.es2015.proxy.d.ts","../node_modules/typescript/lib/lib.es2015.reflect.d.ts","../node_modules/typescript/lib/lib.es2015.symbol.d.ts","../node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts","../node_modules/typescript/lib/lib.decorators.d.ts","../node_modules/typescript/lib/lib.decorators.legacy.d.ts","../src/index.ts"],"fileInfos":[{"version":"f59215c5f1d886b05395ee7aca73e0ac69ddfad2843aa88530e797879d511bad","affectsGlobalScope":true},"45b7ab580deca34ae9729e97c13cfd999df04416a79116c3bfb483804f85ded4",{"version":"9d9885c728913c1d16e0d2831b40341d6ad9a0ceecaabc55209b306ad9c736a5","affectsGlobalScope":true},{"version":"17bea081b9c0541f39dd1ae9bc8c78bdd561879a682e60e2f25f688c0ecab248","affectsGlobalScope":true},{"version":"4443e68b35f3332f753eacc66a04ac1d2053b8b035a0e0ac1d455392b5e243b3","affectsGlobalScope":true},{"version":"ab22100fdd0d24cfc2cc59d0a00fc8cf449830d9c4030dc54390a46bd562e929","affectsGlobalScope":true},{"version":"f7bd636ae3a4623c503359ada74510c4005df5b36de7f23e1db8a5c543fd176b","affectsGlobalScope":true},{"version":"ce691fb9e5c64efb9547083e4a34091bcbe5bdb41027e310ebba8f7d96a98671","affectsGlobalScope":true},{"version":"8d697a2a929a5fcb38b7a65594020fcef05ec1630804a33748829c5ff53640d0","affectsGlobalScope":true},{"version":"0c20f4d2358eb679e4ae8a4432bdd96c857a2960fd6800b21ec4008ec59d60ea","affectsGlobalScope":true},{"version":"36ae84ccc0633f7c0787bc6108386c8b773e95d3b052d9464a99cd9b8795fbec","affectsGlobalScope":true},{"version":"189c0703923150aa30673fa3de411346d727cc44a11c75d05d7cf9ef095daa22","affectsGlobalScope":true},{"version":"782dec38049b92d4e85c1585fbea5474a219c6984a35b004963b00beb1aab538","affectsGlobalScope":true},{"version":"0b0efe6cf1c03e6fcdb6e0d4650267739d78670d17e77310d8e4930d91671622","signature":"b4370986352e9c14688509ded0252bf8daa86bdcff5f2c2f7c8b2f7514fec525"}],"root":[14],"options":{"declaration":true,"esModuleInterop":true,"module":1,"outDir":"./","strict":true,"target":2},"referencedMap":[],"exportedModulesMap":[],"semanticDiagnosticsPerFile":[12,13,4,3,2,5,6,7,8,9,10,11,1,14]},"version":"5.1.6"}

package/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "braintrust",
+  "version": "0.0.1",
+  "description": "SDK for integrating BrainTrust",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc",
+    "test": "jest"
+  },
+  "author": "",
+  "license": "MIT",
+  "devDependencies": {
+    "typescript": "^5.1.6"
+  }
+}

package/src/index.ts

@@ -0,0 +1,112 @@
+/**
+ * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
+ *
+ * @param project The name of the project to create the experiment in.
+ * @param options
+ * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
+ * @param options.description An optional description of the experiment.
+ * @param options.base_experiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
+ * experiment. Otherwise, it will pick an experiment by finding the closest ancestor on the default (e.g. main) branch.
+ * @param options.api_url The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
+ * @param options.api_key The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
+ * key is specified, will prompt the user to login.
+ * @param options.org_name (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
+ * @param options.disable_cache Do not use cached login information.
+ * @returns The experiment object.
+ */
+
+export function init(
+  project: string,
+  {
+    experiment,
+  }: {
+    readonly experiment?: string;
+    readonly description?: string;
+    readonly base_experiment?: string;
+    readonly api_url?: string;
+    readonly api_key?: string;
+    readonly org_name?: string;
+    readonly disable_cache?: boolean;
+  }
+) {
+  // TODO
+  return new Experiment(project);
+}
+
+export class Experiment {
+  private _project: string;
+
+  constructor(project: string) {
+    this._project = project;
+  }
+
+  get project() {
+    return this._project;
+  }
+
+  /**
+   * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
+   *
+   * @param values
+   * @param values.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
+   * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
+   * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
+   * `inputs` should be identical.
+   * @param values.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
+   * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
+   * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
+   * be multiple valid queries that answer a single question.
+   * @param values.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
+   * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
+   * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
+   * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
+   * fine-tune your models.
+   * @param values.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
+   * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
+   * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
+   * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
+   * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
+   * @param values.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
+   * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+   * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+   * JSON-serializable type, but its keys must be strings.
+   * @returns The `id` of the logged event.
+   */
+  log({
+    inputs,
+    output,
+    expected,
+    scores,
+    metadata,
+  }: {
+    readonly inputs: unknown;
+    readonly output: unknown;
+    readonly expected: unknown;
+    readonly scores: Record<string, number>;
+    readonly metadata?: Record<string, unknown>;
+  }): string {
+    // TODO
+    (() => ({ inputs, output, expected, scores, metadata }))();
+
+    return "foo";
+  }
+
+  /**
+   * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
+   *
+   * @param options
+   * @param summarize_scores Whether to summarize the scores. If False, only the metadata will be returned.
+   * @param comparison_experiment_id The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
+   * @returns `ExperimentSummary`
+   */
+  summarize(
+    options:
+      | {
+          readonly summarizeScores?: boolean;
+          readonly comparisonExperimentId?: string;
+        }
+      | undefined = undefined
+  ): string {
+    return "Summary!";
+  }
+}

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+	"incremental": true,
+    "declaration": true,
+    "outDir": "./dist",
+    "lib": ["es6"],
+    "module": "commonjs",
+    "target": "es6",
+    "moduleResolution": "node",
+    "strict": true,
+    "esModuleInterop": true
+  },
+  "include": ["src"],
+  "exclude": ["node_modules/**"]
+}