braintrust 0.0.21 → 0.0.22

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,348 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const esbuild = __importStar(require("esbuild"));
+const chalk_1 = __importDefault(require("chalk"));
+const fs_1 = __importDefault(require("fs"));
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const util_1 = __importDefault(require("util"));
+const fsWalk = __importStar(require("@nodelib/fs.walk"));
+const minimatch_1 = require("minimatch");
+const argparse_1 = require("argparse");
+const uuid_1 = require("uuid");
+const pluralize_1 = __importDefault(require("pluralize"));
+const logger_1 = require("./logger");
+// Re-use the module resolution logic from Jest
+const nodeModulesPaths_1 = __importDefault(require("./jest/nodeModulesPaths"));
+const framework_1 = require("./framework");
+// This requires require
+// https://stackoverflow.com/questions/50822310/how-to-import-package-json-in-typescript
+const { version } = require("../package.json");
+// TODO: This could be loaded from configuration
+const INCLUDE = ["**/*.eval.ts", "**/*.eval.js"];
+const EXCLUDE = ["**/node_modules/**", "**/dist/**", "**/build/**"];
+const OUT_EXT = "js";
+const error = chalk_1.default.bold.red;
+const warning = chalk_1.default.hex("#FFA500"); // Orange color
+function logError(e, verbose) {
+    if (!verbose) {
+        console.error(`${e}`);
+    }
+    else {
+        console.error(e);
+    }
+}
+function evalWithModuleContext(inFile, evalFn) {
+    const modulePaths = [...module.paths];
+    try {
+        module.paths = (0, nodeModulesPaths_1.default)(path_1.default.dirname(inFile), {});
+        return evalFn();
+    }
+    finally {
+        module.paths = modulePaths;
+    }
+}
+function initFile(inFile, outFile) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const ctx = yield esbuild.context(buildOpts(inFile, outFile));
+        return {
+            inFile,
+            outFile,
+            rebuild: () => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    const result = yield ctx.rebuild();
+                    if (!result.outputFiles) {
+                        return {
+                            type: "failure",
+                            error: new Error("No output file generated"),
+                            sourceFile: inFile,
+                        };
+                    }
+                    const moduleText = result.outputFiles[0].text;
+                    const evaluator = evalWithModuleContext(inFile, () => {
+                        globalThis._evals = {};
+                        eval(moduleText);
+                        return Object.assign({}, globalThis._evals);
+                    });
+                    return { type: "success", result, evaluator, sourceFile: inFile };
+                }
+                catch (e) {
+                    return { type: "failure", error: e, sourceFile: inFile };
+                }
+            }),
+            destroy: () => __awaiter(this, void 0, void 0, function* () {
+                yield ctx.dispose();
+            }),
+        };
+    });
+}
+function updateEvaluators(evaluators, buildResults, opts) {
+    for (const result of buildResults) {
+        if (result.type === "failure") {
+            if (opts.verbose) {
+                console.warn(`Failed to compile ${result.sourceFile}`);
+                console.warn(result.error);
+            }
+            else {
+                console.warn(`Failed to compile ${result.sourceFile}: ${result.error.message}`);
+            }
+            continue;
+        }
+        for (const [name, evaluator] of Object.entries(result.evaluator)) {
+            if (evaluators[name] &&
+                (evaluators[name].sourceFile !== result.sourceFile ||
+                    evaluators[name].evaluator !== evaluator)) {
+                console.warn(warning(`Evaluator ${name} already exists (in ${evaluators[name].sourceFile} and ${result.sourceFile}). Will skip ${name} in ${result.sourceFile}.`));
+            }
+            evaluators[name] = {
+                sourceFile: result.sourceFile,
+                evaluator,
+            };
+        }
+    }
+}
+function runOnce(handles, opts) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const buildPromises = Object.values(handles).map((handle) => handle.rebuild());
+        const buildResults = yield Promise.all(buildPromises);
+        const evaluators = {};
+        updateEvaluators(evaluators, buildResults, opts);
+        // TODO: We should probably take login arguments here
+        yield (0, logger_1.login)({
+            apiKey: opts.apiKey,
+            orgName: opts.orgName,
+            apiUrl: opts.apiUrl,
+        });
+        const resultPromises = Object.values(evaluators).map((evaluator) => __awaiter(this, void 0, void 0, function* () {
+            // TODO: For now, use the eval name as the project. However, we need to evolve
+            // the definition of a project and create a new concept called run, so that we
+            // can name the experiment/evaluation within the run the evaluator's name.
+            const logger = yield (0, logger_1.init)(evaluator.evaluator.name);
+            return yield (0, framework_1.runEvaluator)(logger, evaluator.evaluator);
+        }));
+        const allEvalsResults = yield Promise.all(resultPromises);
+        for (const [evaluator, idx] of Object.keys(evaluators).map((k, i) => [
+            k,
+            i,
+        ])) {
+            const { results, summary } = allEvalsResults[idx];
+            const failingResults = results.filter((r) => r.error !== undefined);
+            if (failingResults.length > 0) {
+                // TODO: We may want to support a non-strict mode (and make this the "strict" behavior), so that
+                // users can still log imperfect evaluations. In the meantime, they should handle these cases inside
+                // of their tasks.
+                console.warn(warning(`Evaluator ${evaluator} failed with ${(0, pluralize_1.default)("error", failingResults.length, true)}${!opts.verbose ? " (add --verbose to see the full error)" : ""}. This evaluation ("${evaluator}") will not be fully logged.`));
+                for (const result of failingResults) {
+                    logError(result.error, opts.verbose);
+                }
+            }
+            else {
+                console.log(summary);
+            }
+        }
+    });
+}
+function checkMatch(pathInput, include_patterns, exclude_patterns) {
+    const p = path_1.default.resolve(pathInput);
+    if (include_patterns !== null) {
+        let include = false;
+        for (const pattern of include_patterns) {
+            if ((0, minimatch_1.minimatch)(p, pattern)) {
+                include = true;
+                break;
+            }
+        }
+        if (!include) {
+            return false;
+        }
+    }
+    if (exclude_patterns !== null) {
+        let exclude = false;
+        for (const pattern of exclude_patterns) {
+            if ((0, minimatch_1.minimatch)(p, pattern)) {
+                exclude = true;
+                break;
+            }
+        }
+        return !exclude;
+    }
+    return true;
+}
+function collectFiles(inputPath) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let pathStat = null;
+        try {
+            pathStat = fs_1.default.lstatSync(inputPath);
+        }
+        catch (e) {
+            console.error(error(`Error reading ${inputPath}: ${e}`));
+            process.exit(1);
+        }
+        let files = [];
+        if (!pathStat.isDirectory()) {
+            if (checkMatch(inputPath, INCLUDE, EXCLUDE)) {
+                files.push(inputPath);
+            }
+        }
+        else {
+            const walked = yield util_1.default.promisify(fsWalk.walk)(inputPath, {
+                deepFilter: (entry) => {
+                    return checkMatch(entry.path, null, EXCLUDE);
+                },
+                entryFilter: (entry) => {
+                    return (entry.dirent.isFile() && checkMatch(entry.path, INCLUDE, EXCLUDE));
+                },
+            });
+            files = files.concat(walked.map((entry) => entry.path));
+        }
+        return files;
+    });
+}
+function buildOpts(fileName, outFile) {
+    return {
+        entryPoints: [fileName],
+        bundle: true,
+        outfile: outFile,
+        platform: "node",
+        packages: "external",
+        write: false,
+        // Remove the leading "v" from process.version
+        target: `node${process.version.slice(1)}`,
+    };
+}
+function buildContext(fileName, outFile) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return yield esbuild.context(buildOpts(fileName, outFile));
+    });
+}
+function initializeHandles(args) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const files = {};
+        const inputPaths = args.files.length > 0 ? args.files : ["."];
+        for (const inputPath of inputPaths) {
+            const newFiles = yield collectFiles(inputPath);
+            for (const file of newFiles) {
+                files[path_1.default.resolve(file)] = true;
+            }
+        }
+        // XXX We can probably not even create this dir?
+        let tmpDir = path_1.default.join(os_1.default.tmpdir(), `btevals-${(0, uuid_1.v4)().slice(0, 8)}`);
+        // fs.mkdirSync(tmpDir, { recursive: true });
+        const initPromises = [];
+        for (const file of Object.keys(files)) {
+            const outFile = path_1.default.join(tmpDir, `${path_1.default.basename(file, path_1.default.extname(file))}-${(0, uuid_1.v4)().slice(0, 8)}.${OUT_EXT}`);
+            initPromises.push(initFile(file, outFile));
+        }
+        const handles = {};
+        const initResults = yield Promise.all(initPromises);
+        for (const result of initResults) {
+            handles[result.inFile] = result;
+        }
+        return handles;
+    });
+}
+function run(args) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const handles = yield initializeHandles(args);
+        try {
+            if (args.watch) {
+                throw new Error("Unimplmented");
+            }
+            else {
+                runOnce(handles, {
+                    verbose: args.verbose,
+                    apiKey: args.api_key,
+                    orgName: args.org_name,
+                    apiUrl: args.api_url,
+                });
+            }
+        }
+        finally {
+            // ESBuild can freeze up if you do not clean up the handles properly
+            for (const handle of Object.values(handles)) {
+                yield handle.destroy();
+            }
+        }
+    });
+}
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const [, ...args] = process.argv;
+        const parser = new argparse_1.ArgumentParser({
+            description: "Argparse example",
+        });
+        parser.add_argument("-v", "--version", { action: "version", version });
+        const parentParser = new argparse_1.ArgumentParser({ add_help: false });
+        parentParser.add_argument("--verbose", { action: "store_true" });
+        const subparser = parser.add_subparsers({
+            required: true,
+        });
+        const parser_run = subparser.add_parser("run", {
+            help: "Run evals locally",
+            parents: [parentParser],
+        });
+        parser_run.add_argument("--api-key", {
+            help: "Specify a braintrust api key. If the parameter is not specified, the BRAINTRUST_API_KEY environment variable will be used.",
+        });
+        parser_run.add_argument("--org-name", {
+            help: "The name of a specific organization to connect to. This is useful if you belong to multiple.",
+        });
+        parser_run.add_argument("--api-url", {
+            help: "Specify a custom braintrust api url. Defaults to https://www.braintrustdata.com. This is only necessary if you are using an experimental version of BrainTrust",
+        });
+        parser_run.add_argument("--watch", {
+            action: "store_true",
+            help: "Watch files for changes and rerun evals when changes are detected",
+        });
+        parser_run.add_argument("files", {
+            nargs: "*",
+            help: "A list of files or directories to run. If no files are specified, the current directory is used.",
+        });
+        parser_run.set_defaults({ func: run });
+        const parsed = parser.parse_args();
+        try {
+            yield parsed.func(parsed);
+        }
+        catch (e) {
+            logError(e, parsed.verbose);
+            process.exit(1);
+        }
+    });
+}
+main();

package/dist/framework.d.ts

@@ -0,0 +1,42 @@
+import { Experiment } from "./logger";
+import { Score } from "autoevals";
+export type Metadata = Record<string, unknown>;
+export interface DatasetRecord<Input, Output> {
+    input: Input;
+    expected?: Output;
+    metadata?: Metadata;
+}
+export type EvalData<Input, Output> = string | (() => DatasetRecord<Input, Output>[]) | (() => Promise<DatasetRecord<Input, Output>[]>);
+export type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
+export interface EvalHooks {
+    meta: (info: Record<string, unknown>) => void;
+}
+export type EvalScorerArgs<Input, Output> = DatasetRecord<Input, Output> & {
+    output: Output;
+};
+export type EvalScorer<Input, Output> = ((args: EvalScorerArgs<Input, Output>) => Score) | ((args: EvalScorerArgs<Input, Output>) => Promise<Score>);
+export interface Evaluator<Input, Output> {
+    task: EvalTask<Input, Output>;
+    scores: EvalScorer<Input, Output>[];
+    data: EvalData<Input, Output>;
+}
+export type EvaluatorDef<Input, Output> = {
+    name: string;
+} & Evaluator<Input, Output>;
+export type EvaluatorFile = {
+    [evaluator: string]: EvaluatorDef<any, any>;
+};
+declare global {
+    var _evals: EvaluatorFile;
+}
+export declare function Eval<Input, Output>(name: string, evaluator: Evaluator<Input, Output>): void;
+export declare function getLoadedEvals(): EvaluatorFile;
+export declare function runEvaluator(experiment: Experiment, evaluator: EvaluatorDef<unknown, unknown>): Promise<{
+    results: {
+        output: any;
+        metadata: Metadata;
+        scores: Record<string, number>;
+        error: unknown;
+    }[];
+    summary: import("./logger").ExperimentSummary;
+}>;

package/dist/framework.js

@@ -0,0 +1,104 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runEvaluator = exports.getLoadedEvals = exports.Eval = void 0;
+globalThis._evals = {};
+function Eval(name, evaluator) {
+    if (_evals[name]) {
+        throw new Error(`Evaluator ${name} already exists`);
+    }
+    _evals[name] = Object.assign({ name }, evaluator);
+}
+exports.Eval = Eval;
+function getLoadedEvals() {
+    return _evals;
+}
+exports.getLoadedEvals = getLoadedEvals;
+function runEvaluator(experiment, evaluator) {
+    return __awaiter(this, void 0, void 0, function* () {
+        console.log(`Running evaluator ${evaluator.name}`);
+        if (typeof evaluator.data === "string") {
+            throw new Error("Unimplemented: string data paths");
+        }
+        const dataResult = evaluator.data();
+        let data = null;
+        if (dataResult instanceof Promise) {
+            data = yield dataResult;
+        }
+        else {
+            data = dataResult;
+        }
+        const evals = data.map((datum) => __awaiter(this, void 0, void 0, function* () {
+            let metadata = {};
+            let output = undefined;
+            let error = undefined;
+            let scores = {};
+            try {
+                const meta = (o) => (metadata = Object.assign(Object.assign({}, metadata), o));
+                const outputResult = evaluator.task(datum.input, {
+                    meta,
+                });
+                if (outputResult instanceof Promise) {
+                    output = yield outputResult;
+                }
+                else {
+                    output = outputResult;
+                }
+                const scoringArgs = Object.assign(Object.assign({}, datum), { output });
+                const scoreResults = yield Promise.all(evaluator.scores.map((score) => __awaiter(this, void 0, void 0, function* () {
+                    const scoreResult = score(scoringArgs);
+                    if (scoreResult instanceof Promise) {
+                        return yield scoreResult;
+                    }
+                    else {
+                        return scoreResult;
+                    }
+                })));
+                const scoreMetadata = {};
+                for (const scoreResult of scoreResults) {
+                    scores[scoreResult.name] = scoreResult.score;
+                    const metadata = Object.assign(Object.assign({}, scoreResult.metadata), { error: scoreResult.error });
+                    if (Object.keys(metadata).length > 0) {
+                        scoreMetadata[scoreResult.name] = metadata;
+                    }
+                }
+                if (Object.keys(scoreMetadata).length > 0) {
+                    meta({ scores: scoreMetadata });
+                }
+            }
+            catch (e) {
+                error = e;
+            }
+            experiment.log({
+                // TODO We should rename this from inputs -> input in the logger, etc.
+                // https://github.com/braintrustdata/braintrust/issues/217
+                inputs: datum.input,
+                metadata: datum.metadata,
+                expected: datum.expected,
+                output,
+                scores,
+            });
+            return {
+                output,
+                metadata,
+                scores,
+                error,
+            };
+        }));
+        const results = yield Promise.all(evals);
+        const summary = yield experiment.summarize();
+        return {
+            results,
+            summary,
+        };
+    });
+}
+exports.runEvaluator = runEvaluator;

package/dist/gitutil.js

@@ -38,15 +38,22 @@ function getBaseBranch(remote = undefined) {
                 // fail with a cryptic error message.
                 throw new Error("No remote found");
             }
-            const remoteInfo = yield git.remote(["show", remoteName]);
-            if (!remoteInfo) {
-                throw new Error(`Could not find remote ${remoteName}`);
+            let branch = null;
+            try {
+                const remoteInfo = yield git.remote(["show", remoteName]);
+                if (!remoteInfo) {
+                    throw new Error(`Could not find remote ${remoteName}`);
+                }
+                const match = remoteInfo.match(/\s*HEAD branch:\s*(.*)$/m);
+                if (!match) {
+                    throw new Error(`Could not find HEAD branch in remote ${remoteName}`);
+                }
+                branch = match[1];
             }
-            const match = remoteInfo.match(/\s*HEAD branch:\s*(.*)$/m);
-            if (!match) {
-                throw new Error(`Could not find HEAD branch in remote ${remoteName}`);
+            catch (_b) {
+                branch = "main";
             }
-            _baseBranch = { remote: remoteName, branch: match[1] };
+            _baseBranch = { remote: remoteName, branch };
         }
         return _baseBranch;
     });