braintrust 0.0.140 → 0.0.141-dev

package/package.json

@@ -1,31 +1,33 @@
 {
   "name": "braintrust",
-  "version": "0.0.140",
+  "version": "0.0.141-dev",
   "description": "SDK for integrating Braintrust",
   "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
   "browser": {
     "./dist/index.js": "./dist/browser.js",
-    "./dist/index.d.ts": "./dist/browser.d.ts"
+    "./dist/index.d.ts": "./dist/browser.d.ts",
+    "./dist/index.mjs": "./dist/browser.mjs",
+    "./dist/index.d.mts": "./dist/browser.d.mts"
   },
   "bin": {
     "braintrust": "./dist/cli.js"
   },
-  "scripts": {
-    "build": "run-p build:*",
-    "build:node": "esbuild --platform=node --bundle src/index.ts --outfile=dist/index.js",
-    "build:cli": "esbuild --platform=node --bundle src/cli.ts --outfile=dist/cli.js --external:esbuild",
-    "build:browser": "esbuild --bundle src/browser.ts --outfile=dist/browser.js --format=esm",
-    "build:declarations": "tsc --declaration  --emitDeclarationOnly --outDir dist",
-    "clean": "rm -r dist/*",
-    "watch": "run-p watch:*",
-    "watch:node": "esbuild --platform=node --bundle src/index.ts --outfile=dist/index.js --watch=forever",
-    "watch:cli": "esbuild --platform=node --bundle src/cli.ts --outfile=dist/cli.js --watch=forever --external:esbuild",
-    "watch:browser": "esbuild --bundle src/browser.ts --outfile=dist/browser.js --watch=forever --format=esm",
-    "watch:declarations": "tsc --declaration  --emitDeclarationOnly --outDir dist --watch",
-    "docs": "npx typedoc --options typedoc.json src/index.ts",
-    "test": "jest",
-    "prepublishOnly": "../../scripts/node_prepublish_sdk.py",
-    "postpublish": "../../scripts/node_postpublish_sdk.py"
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "module": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "./ai-sdk": {
+      "types": "./ai-sdk/dist/index.d.ts",
+      "import": "./ai-sdk/dist/index.mjs",
+      "module": "./ai-sdk/dist/index.mjs",
+      "require": "./ai-sdk/dist/index.js"
+    }
   },
   "author": "",
   "license": "MIT",
@@ -42,28 +44,43 @@
     "@types/uuid": "^9.0.7",
     "async": "^3.2.5",
     "autoevals": "^0.0.69",
-    "jest": "^29.7.0",
     "npm-run-all": "^4.1.5",
     "ts-jest": "^29.1.4",
+    "tsup": "^8.0.1",
     "typedoc": "^0.25.4",
     "typedoc-plugin-markdown": "^3.17.1",
-    "typescript": "^5.3.3"
+    "typescript": "^5.3.3",
+    "vite-tsconfig-paths": "^4.3.2",
+    "vitest": "^1.6.0"
   },
   "dependencies": {
-    "@ai-sdk/provider": "^0.0.6",
-    "@braintrust/core": "0.0.44",
+    "@ai-sdk/provider": "^0.0.11",
     "@next/env": "^14.2.3",
+    "@vercel/functions": "^1.0.2",
+    "ai": "^3.2.16",
     "argparse": "^2.0.1",
     "chalk": "^4.1.2",
     "cli-progress": "^3.12.0",
     "dotenv": "^16.4.5",
     "esbuild": "^0.18.20",
+    "eventsource-parser": "^1.1.2",
     "graceful-fs": "^4.2.11",
     "minimatch": "^9.0.3",
     "mustache": "^4.2.0",
     "pluralize": "^8.0.0",
     "simple-git": "^3.21.0",
     "uuid": "^9.0.1",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "@braintrust/core": "0.0.45"
+  },
+  "peerDependencies": {
+    "zod": "^3.0.0"
+  },
+  "scripts": {
+    "build": "tsup",
+    "watch": "tsup --watch",
+    "clean": "rm -r dist/*",
+    "docs": "npx typedoc --options typedoc.json src/index.ts",
+    "test": "vitest run"
   }
 }

package/tsup.config.ts

@@ -0,0 +1,23 @@
+import { defineConfig } from "tsup";
+
+export default defineConfig([
+  {
+    entry: ["src/index.ts"],
+    format: ["cjs", "esm"],
+    outDir: "dist",
+    external: ["zod"],
+    dts: true,
+  },
+  {
+    entry: ["src/browser.ts"],
+    format: ["cjs", "esm"],
+    outDir: "dist",
+    dts: true,
+  },
+  {
+    entry: ["src/cli.ts"],
+    format: ["cjs"],
+    outDir: "dist",
+    external: ["esbuild"],
+  },
+]);

package/turbo.json

@@ -0,0 +1,8 @@
+{
+  "extends": ["//"],
+  "pipeline": {
+    "build": {
+      "outputs": ["**/dist/**"]
+    }
+  }
+}

package/vitest.config.js

@@ -0,0 +1,6 @@
+import tsconfigPaths from "vite-tsconfig-paths";
+
+const config = {
+  plugins: [tsconfigPaths()],
+};
+export default config;

package/dist/browser-config.d.ts

@@ -1,6 +0,0 @@
-/// <reference types="node" />
-import type { AsyncLocalStorage as NodeAsyncLocalStorage } from "async_hooks";
-declare global {
-    var AsyncLocalStorage: typeof NodeAsyncLocalStorage;
-}
-export declare function configureBrowser(): void;

package/dist/cli.d.ts

@@ -1,35 +0,0 @@
-#!/usr/bin/env node
-import * as esbuild from "esbuild";
-import { EvaluatorDef, EvaluatorFile, ReporterDef } from "./framework";
-interface BuildSuccess {
-    type: "success";
-    result: esbuild.BuildResult;
-    evaluator: EvaluatorFile;
-    sourceFile: string;
-}
-interface BuildFailure {
-    type: "failure";
-    error: Error;
-    sourceFile: string;
-}
-type BuildResult = BuildSuccess | BuildFailure;
-export interface FileHandle {
-    inFile: string;
-    outFile: string;
-    bundleFile?: string;
-    rebuild: () => Promise<BuildResult>;
-    bundle: () => Promise<esbuild.BuildResult>;
-    watch: () => void;
-    destroy: () => Promise<void>;
-}
-export interface EvaluatorState {
-    evaluators: {
-        sourceFile: string;
-        evaluator: EvaluatorDef<any, any, any, any>;
-        reporter: string | ReporterDef<any> | undefined;
-    }[];
-    reporters: {
-        [reporter: string]: ReporterDef<any>;
-    };
-}
-export {};

package/dist/framework.d.ts

@@ -1,180 +0,0 @@
-import chalk from "chalk";
-import { NOOP_SPAN, Experiment, ExperimentSummary, Span, EvalCase, BaseMetadata, DefaultMetadataType, currentSpan, BraintrustState } from "./logger";
-import { Score } from "@braintrust/core";
-import { ProgressReporter } from "./progress";
-export type BaseExperiment<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
-    _type: "BaseExperiment";
-    _phantom?: [Input, Expected, Metadata];
-    name?: string;
-};
-/**
- * Use this to specify that the dataset should actually be the data from a previous (base) experiment.
- * If you do not specify a name, Braintrust will automatically figure out the best base experiment to
- * use based on your git history (or fall back to timestamps).
- *
- * @param options
- * @param options.name The name of the base experiment to use. If unspecified, Braintrust will automatically figure out the best base
- * using your git history (or fall back to timestamps).
- * @returns
- */
-export declare function BaseExperiment<Input = unknown, Expected = unknown, Metadata extends BaseMetadata = DefaultMetadataType>(options?: {
-    name?: string;
-}): BaseExperiment<Input, Expected, Metadata>;
-export type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
-export type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
-export interface EvalHooks {
-    meta: (info: Record<string, unknown>) => void;
-    span: Span;
-}
-export type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
-    output: Output;
-};
-type OneOrMoreScores = Score | number | null | Array<Score>;
-export type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreScores | Promise<OneOrMoreScores>;
-export type EvalResult<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
-    output: Output;
-    scores: Record<string, number | null>;
-    error: unknown;
-};
-export interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> {
-    /**
-     * A function that returns a list of inputs, expected outputs, and metadata.
-     */
-    data: EvalData<Input, Expected, Metadata>;
-    /**
-     * A function that takes an input and returns an output.
-     */
-    task: EvalTask<Input, Output>;
-    /**
-     * A set of functions that take an input, output, and expected value and return a score.
-     */
-    scores: EvalScorer<Input, Output, Expected, Metadata>[];
-    /**
-     * An optional name for the experiment.
-     */
-    experimentName?: string;
-    /**
-     * The number of times to run the evaluator per input. This is useful for evaluating applications that
-     * have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the
-     * variance in the results.
-     */
-    trialCount?: number;
-    /**
-     * Optional additional metadata for the experiment.
-     */
-    metadata?: Record<string, unknown>;
-    /**
-     * Whether the experiment should be public. Defaults to false.
-     */
-    isPublic?: boolean;
-    /**
-     * Whether to update an existing experiment with `experiment_name` if one exists. Defaults to false.
-     */
-    update?: boolean;
-    /**
-     * The duration, in milliseconds, after which to time out the evaluation.
-     * Defaults to undefined, in which case there is no timeout.
-     */
-    timeout?: number;
-    /**
-     * The maximum number of tasks/scorers that will be run concurrently.
-     * Defaults to undefined, in which case there is no max concurrency.
-     */
-    maxConcurrency?: number;
-    /**
-     * If specified, uses the given project ID instead of the evaluator's name to identify the project.
-     */
-    projectId?: string;
-    /**
-     * If specified, uses the logger state to initialize Braintrust objects. If unspecified, falls back
-     * to the global state (initialized using your API key).
-     */
-    state?: BraintrustState;
-}
-export type EvalResultWithSummary<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
-    summary: ExperimentSummary;
-    results: EvalResult<Input, Output, Expected, Metadata>[];
-};
-export interface ReporterOpts {
-    verbose: boolean;
-    jsonl: boolean;
-}
-export interface ReporterBody<EvalReport> {
-    /**
-     * A function that takes an evaluator and its result and returns a report.
-     *
-     * @param evaluator
-     * @param result
-     * @param opts
-     */
-    reportEval(evaluator: EvaluatorDef<any, any, any, any>, result: EvalResultWithSummary<any, any, any, any>, opts: ReporterOpts): Promise<EvalReport> | EvalReport;
-    /**
-     * A function that takes all evaluator results and returns a boolean indicating
-     * whether the run was successful. If you return false, the `braintrust eval`
-     * command will exit with a non-zero status code.
-     *
-     * @param reports
-     */
-    reportRun(reports: EvalReport[]): boolean | Promise<boolean>;
-}
-export type ReporterDef<EvalReport> = {
-    name: string;
-} & ReporterBody<EvalReport>;
-export type EvaluatorDef<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
-    projectName: string;
-    evalName: string;
-} & Evaluator<Input, Output, Expected, Metadata>;
-export type EvaluatorFile = {
-    evaluators: {
-        [evalName: string]: {
-            evaluator: EvaluatorDef<any, any, any, any>;
-            reporter?: ReporterDef<unknown> | string;
-        };
-    };
-    reporters: {
-        [reporterName: string]: ReporterDef<unknown>;
-    };
-};
-export type SpanContext = {
-    currentSpan: typeof currentSpan;
-    NOOP_SPAN: typeof NOOP_SPAN;
-};
-declare global {
-    var _evals: EvaluatorFile;
-    var _spanContext: SpanContext | undefined;
-    var _lazy_load: boolean;
-}
-export interface EvalOptions<EvalReport> {
-    reporter?: ReporterDef<EvalReport> | string;
-    onStart?: (metadata: Omit<ExperimentSummary, "scores" | "metrics">) => void;
-}
-export declare function Eval<Input, Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, EvalReport = boolean>(name: string, evaluator: Evaluator<Input, Output, Expected, Metadata>, reporterOrOpts?: ReporterDef<EvalReport> | string | EvalOptions<EvalReport>): Promise<EvalResultWithSummary<Input, Output, Expected, Metadata>>;
-export declare function Reporter<EvalReport>(name: string, reporter: ReporterBody<EvalReport>): ReporterDef<EvalReport>;
-export declare function getLoadedEvals(): EvaluatorFile;
-export interface Filter {
-    path: string[];
-    pattern: RegExp;
-}
-export declare function serializeJSONWithPlainString(v: any): string;
-export declare function deserializePlainStringAsJSON(s: string): {
-    value: any;
-    error: undefined;
-} | {
-    value: string;
-    error: unknown;
-};
-export declare function parseFilters(filters: string[]): Filter[];
-export declare function scorerName(scorer: EvalScorer<any, any, any, any>, scorer_idx: number): string;
-export declare function runEvaluator(experiment: Experiment | null, evaluator: EvaluatorDef<any, any, any, any>, progressReporter: ProgressReporter, filters: Filter[]): Promise<EvalResultWithSummary<any, any, any, any>>;
-export declare const error: chalk.Chalk;
-export declare const warning: chalk.Chalk;
-export declare function logError(e: unknown, verbose: boolean): void;
-export declare function buildLocalSummary(evaluator: EvaluatorDef<any, any, any, any>, results: EvalResult<any, any, any, any>[]): ExperimentSummary;
-export declare function reportFailures<Input, Output, Expected, Metadata extends BaseMetadata>(evaluator: EvaluatorDef<Input, Output, Expected, Metadata>, failingResults: EvalResult<Input, Output, Expected, Metadata>[], { verbose, jsonl }: ReporterOpts): void;
-/**
- * The default reporter for Braintrust evaluations. This reporter will log the results
- * of each evaluation to the console, and will return false (i.e. fail) if any of the
- * evaluations return an error.
- */
-export declare const defaultReporter: ReporterDef<boolean>;
-export {};

package/dist/framework.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/functions.d.ts

@@ -1,15 +0,0 @@
-import { EvaluatorState, FileHandle } from "./cli";
-import { Experiment } from "./logger";
-import * as esbuild from "esbuild";
-export type EvaluatorMap = Record<string, {
-    evaluator: EvaluatorState["evaluators"][number];
-    experiment: Experiment;
-}>;
-export declare function uploadEvalBundles({ experimentIdToEvaluator, bundlePromises, handles, verbose, }: {
-    experimentIdToEvaluator: EvaluatorMap;
-    bundlePromises: {
-        [k: string]: Promise<esbuild.BuildResult<esbuild.BuildOptions>>;
-    };
-    handles: Record<string, FileHandle>;
-    verbose: boolean;
-}): Promise<void>;

package/dist/gitutil.d.ts

@@ -1,17 +0,0 @@
-import { GitMetadataSettings } from "@braintrust/core";
-/**
- * Information about the current HEAD of the repo.
- */
-export declare function currentRepo(): Promise<import("simple-git").SimpleGit | null>;
-export declare function getPastNAncestors(n?: number, remote?: string | undefined): Promise<string[]>;
-export declare function getRepoInfo(settings?: GitMetadataSettings): Promise<{
-    commit?: string | null | undefined;
-    branch?: string | null | undefined;
-    tag?: string | null | undefined;
-    dirty?: boolean | null | undefined;
-    author_name?: string | null | undefined;
-    author_email?: string | null | undefined;
-    commit_message?: string | null | undefined;
-    commit_time?: string | null | undefined;
-    git_diff?: string | null | undefined;
-} | undefined>;

package/dist/isomorph.d.ts

@@ -1,27 +0,0 @@
-import { GitMetadataSettings, RepoInfo } from "@braintrust/core";
-export interface CallerLocation {
-    caller_functionname: string;
-    caller_filename: string;
-    caller_lineno: number;
-}
-export interface IsoAsyncLocalStorage<T> {
-    enterWith(store: T): void;
-    run<R>(store: T | undefined, callback: () => R): R;
-    getStore(): T | undefined;
-}
-export interface Common {
-    getRepoInfo: (settings?: GitMetadataSettings) => Promise<RepoInfo | undefined>;
-    getPastNAncestors: () => Promise<string[]>;
-    getEnv: (name: string) => string | undefined;
-    getCallerLocation: () => CallerLocation | undefined;
-    newAsyncLocalStorage: <T>() => IsoAsyncLocalStorage<T>;
-    processOn: (event: string, handler: (code: any) => void) => void;
-    pathJoin?: (...args: string[]) => string;
-    pathDirname?: (path: string) => string;
-    mkdir?: (path: string, opts?: {
-        recursive?: boolean;
-    }) => Promise<string | undefined>;
-    writeFile?: (filename: string, data: string) => Promise<void>;
-}
-declare const iso: Common;
-export default iso;

package/dist/jest/nodeModulesPaths.d.ts

@@ -1,15 +0,0 @@
-/**
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- *
- * Adapted from: https://github.com/substack/node-resolve
- */
-type NodeModulesPathsOptions = {
-    moduleDirectory?: Array<string>;
-    paths?: Array<string>;
-};
-export default function nodeModulesPaths(basedir: string, options: NodeModulesPathsOptions): Array<string>;
-export declare const GlobalPaths: string[];
-export {};

package/dist/jest/tryRealpath.d.ts

@@ -1,7 +0,0 @@
-/**
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-export default function tryRealpath(path: string): string;

package/dist/node.d.ts

@@ -1 +0,0 @@
-export declare function configureNode(): void;

package/dist/progress.d.ts

@@ -1,18 +0,0 @@
-export interface ProgressReporter {
-    start: (name: string, total: number) => void;
-    stop: () => void;
-    increment: (name: string) => void;
-}
-export declare class SimpleProgressReporter {
-    start(name: string, _total: number): void;
-    stop(): void;
-    increment(_name: string): void;
-}
-export declare class BarProgressReporter {
-    private multiBar;
-    private bars;
-    constructor();
-    start(name: string, total: number): void;
-    stop(): void;
-    increment(name: string): void;
-}

package/dist/stackutil.d.ts

@@ -1,8 +0,0 @@
-import { CallerLocation } from "./isomorph";
-export interface StackTraceEntry {
-    functionName: string;
-    fileName: string;
-    lineNo: number;
-}
-export declare function getStackTrace(): StackTraceEntry[];
-export declare function getCallerLocation(): CallerLocation | undefined;