braintrust 0.0.21 → 0.0.23

package/dist/index.d.ts

@@ -32,205 +32,5 @@
  *
  * @module braintrust
  */
-export declare class Project {
-    name: string;
-    id: string;
-    org_id: string;
-    constructor(name: string, id: string, org_id: string);
-}
-/**
- * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
- *
- * @param project The name of the project to create the experiment in.
- * @param options Additional options for configuring init().
- * @param options.experiment The name of the experiment to create. If not specified, a name will be generated automatically.
- * @param options.description An optional description of the experiment.
- * @param options.update If the experiment already exists, continue logging to it.
- * @param options.baseExperiment An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this
- * experiment. Otherwise, it will pick an experiment by finding the closest ancestor on the default (e.g. main) branch.
- * @param options.apiUrl The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
- * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
- * key is specified, will prompt the user to login.
- * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
- * @param options.disableCache Do not use cached login information.
- * @returns The newly created Experiment.
- */
-export declare function init(project: string, options?: {
-    readonly experiment?: string;
-    readonly description?: string;
-    readonly update?: boolean;
-    readonly baseExperiment?: string;
-    readonly apiUrl?: string;
-    readonly apiKey?: string;
-    readonly orgName?: string;
-    readonly disableCache?: boolean;
-}): Promise<Experiment>;
-/**
- * Log into BrainTrust. This will prompt you for your API token, which you can find at
- * https://www.braintrustdata.com/app/token. This method is called automatically by `init()`.
- *
- * @param options Options for configuring login().
- * @param options.apiUrl The URL of the BrainTrust API. Defaults to https://www.braintrustdata.com.
- * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API
- * key is specified, will prompt the user to login.
- * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
- * @param options.disableCache Do not use cached login information.
- * @param options.forceLogin Login again, even if you have already logged in (by default, this function will exit quickly if you have already logged in)
- */
-export declare function login(options?: {
-    apiUrl?: string;
-    apiKey?: string;
-    orgName?: string;
-    disableCache?: boolean;
-    forceLogin?: boolean;
-}): Promise<void>;
-/**
- * Log a single event to the current experiment. The event will be batched and uploaded behind the scenes.
- *
- * @param event The event to log.
- * @param event.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
- * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
- * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
- * `inputs` should be identical.
- * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
- * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
- * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
- * be multiple valid queries that answer a single question.
- * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
- * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
- * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
- * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
- * fine-tune your models.
- * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
- * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
- * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
- * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
- * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
- * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
- * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
- * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
- * JSON-serializable type, but its keys must be strings.
- * @param event.id (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
- * @returns The `id` of the logged event.
- */
-export declare function log(options: {
-    readonly inputs: unknown;
-    readonly output: unknown;
-    readonly expected: unknown;
-    readonly scores: Record<string, number>;
-    readonly metadata?: Record<string, unknown>;
-    readonly id?: string;
-}): string;
-/**
- * Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata.
- *
- * @param options Options for summarizing the experiment.
- * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
- * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
- * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
- */
-export declare function summarize(options?: {
-    readonly summarizeScores?: boolean;
-    readonly comparisonExperimentId?: string;
-}): Promise<ExperimentSummary>;
-/**
- * An experiment is a collection of logged events, such as model inputs and outputs, which represent
- * a snapshot of your application at a particular point in time. An experiment is meant to capture more
- * than just the model you use, and includes the data you use to test, pre- and post- processing code,
- * comparison metrics (scores), and any other metadata you want to include.
- *
- * Experiments are associated with a project, and two experiments are meant to be easily comparable via
- * their `inputs`. You can change the attributes of the experiments in a project (e.g. scoring functions)
- * over time, simply by changing what you log.
- *
- * You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
- */
-export declare class Experiment {
-    readonly project: Project;
-    readonly id: string;
-    readonly name: string;
-    readonly user_id: string;
-    private logger;
-    constructor(project: Project, id: string, name: string, user_id: string);
-    /**
-     * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
-     *
-     * @param event The event to log.
-     * @param event.inputs The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
-     * BrainTrust will use the `inputs` to know whether two test casess are the same between experiments, so they should
-     * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
-     * `inputs` should be identical.
-     * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
-     * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
-     * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
-     * be multiple valid queries that answer a single question.
-     * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
-     * determine if your `output` value is correct or not. BrainTrust currently does not compare `output` to `expected` for
-     * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
-     * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
-     * fine-tune your models.
-     * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
-     * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
-     * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
-     * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
-     * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
-     * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
-     * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
-     * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
-     * JSON-serializable type, but its keys must be strings.
-     * @param event.id (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
-     * @returns The `id` of the logged event.
-     */
-    log({ inputs, output, expected, scores, metadata, id, }: {
-        readonly inputs: unknown;
-        readonly output: unknown;
-        readonly expected: unknown;
-        readonly scores: Record<string, number>;
-        readonly metadata?: Record<string, unknown>;
-        readonly id?: string;
-    }): string;
-    /**
-     * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
-     *
-     * @param options Options for summarizing the experiment.
-     * @param options.summarizeScores Whether to summarize the scores. If False, only the metadata will be returned.
-     * @param options.comparisonExperimentId The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used.
-     * @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
-     */
-    summarize(options?: {
-        readonly summarizeScores?: boolean;
-        readonly comparisonExperimentId?: string;
-    }): Promise<ExperimentSummary>;
-}
-/**
- * Summary of a score's performance.
- * @property name Name of the score.
- * @property score Average score across all examples.
- * @property diff Difference in score between the current and reference experiment.
- * @property improvements Number of improvements in the score.
- * @property regressions Number of regressions in the score.
- */
-export interface ScoreSummary {
-    name: string;
-    score: number;
-    diff: number;
-    improvements: number;
-    regressions: number;
-}
-/**
- * Summary of an experiment's scores and metadata.
- * @property projectName Name of the project that the experiment belongs to.
- * @property experimentName Name of the experiment.
- * @property projectUrl URL to the project's page in the BrainTrust app.
- * @property experimentUrl URL to the experiment's page in the BrainTrust app.
- * @property comparisonExperimentName The experiment scores are baselined against.
- * @property scores Summary of the experiment's scores.
- */
-export interface ExperimentSummary {
-    projectName: string;
-    experimentName: string;
-    projectUrl: string;
-    experimentUrl: string;
-    comparisonExperimentName: string | undefined;
-    scores: Record<string, ScoreSummary> | undefined;
-}
+export * from "./logger";
+export { Evaluator, EvalTask, Eval, EvalScorerArgs } from "./framework";