braintrust 0.0.60 → 0.0.62

package/dist/framework.d.ts

@@ -1,4 +1,5 @@
-import { Experiment, Span } from "./logger";
+import chalk from "chalk";
+import { Experiment, ExperimentSummary, InitOptions, Span } from "./logger";
 import { Score } from "autoevals";
 import { ProgressReporter } from "./progress";
 export type Metadata = Record<string, unknown>;
@@ -17,28 +18,39 @@ export type EvalScorerArgs<Input, Output> = EvalCase<Input, Output> & {
     output: Output;
 };
 export type EvalScorer<Input, Output> = ((args: EvalScorerArgs<Input, Output>) => Score) | ((args: EvalScorerArgs<Input, Output>) => Promise<Score>);
+/**
+ * Additional metadata for the eval definition, such as experiment name.
+ */
+export interface EvalMetadata {
+    experimentName?: string;
+}
+export declare function evalMetadataToInitOptions(metadata: EvalMetadata | undefined): InitOptions;
 /**
  * An evaluator is a collection of functions that can be used to evaluate a model.
  * It consists of:
  * - `data`, a function that returns a list of inputs, expected outputs, and metadata
  * - `task`, a function that takes an input and returns an output
  * - `scores`, a set of functions that take an input, output, and expected value and return a score
+ * - `metadata`, optional additional metadata for the eval definition, such as experiment name.
  */
 export interface Evaluator<Input, Output> {
     data: EvalData<Input, Output>;
     task: EvalTask<Input, Output>;
     scores: EvalScorer<Input, Output>[];
+    metadata?: EvalMetadata;
 }
 export type EvaluatorDef<Input, Output> = {
-    name: string;
+    projectName: string;
+    evalName: string;
 } & Evaluator<Input, Output>;
 export type EvaluatorFile = {
-    [evaluator: string]: EvaluatorDef<any, any>;
+    [evalName: string]: EvaluatorDef<any, any>;
 };
 declare global {
     var _evals: EvaluatorFile;
+    var _lazy_load: boolean;
 }
-export declare function Eval<Input, Output>(name: string, evaluator: Evaluator<Input, Output>): void;
+export declare function Eval<Input, Output>(name: string, evaluator: Evaluator<Input, Output>): Promise<void | ExperimentSummary>;
 export declare function getLoadedEvals(): EvaluatorFile;
 export interface Filter {
     path: string[];
@@ -60,5 +72,15 @@ export declare function runEvaluator(experiment: Experiment | null, evaluator: E
         scores: Record<string, number>;
         error: unknown;
     }[];
-    summary: import("./logger").ExperimentSummary | null;
+    summary: ExperimentSummary | null;
 }>;
+export declare const error: chalk.Chalk;
+export declare const warning: chalk.Chalk;
+export declare function logError(e: unknown, verbose: boolean): void;
+export declare function reportEvaluatorResult(evaluatorName: string | number, evaluatorResult: {
+    results: {
+        scores: Record<string, number>;
+        error: unknown;
+    }[];
+    summary: unknown;
+}, verbose: boolean): void;

package/dist/index.d.ts

@@ -33,4 +33,4 @@
  * @module braintrust
  */
 export * from "./logger";
-export { Evaluator, EvalTask, Eval, EvalScorerArgs } from "./framework";
+export { Evaluator, EvalTask, Eval, EvalMetadata, EvalScorerArgs } from "./framework";