braintrust 0.0.113 → 0.0.114

package/dist/framework.d.ts

@@ -29,8 +29,14 @@ export interface EvalHooks {
 export type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;
 };
-type ScoreValue = Score | number | null;
-export type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => ScoreValue | Promise<ScoreValue>;
+type ScoreValue = Score | number;
+type OneOrMoreScores = ScoreValue | Array<ScoreValue> | null;
+export type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => OneOrMoreScores | Promise<OneOrMoreScores>;
+export type EvalResult<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
+    output: Output;
+    scores: Record<string, number | null>;
+    error: unknown;
+};
 export interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> {
     /**
      * A function that returns a list of inputs, expected outputs, and metadata.
@@ -63,18 +69,56 @@ export interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadat
      */
     isPublic?: boolean;
 }
+export type EvalResultWithSummary<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
+    summary: ExperimentSummary;
+    results: EvalResult<Input, Output, Expected, Metadata>[];
+};
+export interface ReporterOpts {
+    verbose: boolean;
+    jsonl: boolean;
+}
+export interface ReporterBody<EvalReport> {
+    /**
+     * A function that takes an evaluator and its result and returns a report.
+     *
+     * @param evaluator
+     * @param result
+     * @param opts
+     */
+    reportEval(evaluator: EvaluatorDef<any, any, any, any>, result: EvalResultWithSummary<any, any, any, any>, opts: ReporterOpts): Promise<EvalReport> | EvalReport;
+    /**
+     * A function that takes all evaluator results and returns a boolean indicating
+     * whether the run was successful. If you return false, the `braintrust eval`
+     * command will exit with a non-zero status code.
+     *
+     * @param reports
+     */
+    reportRun(reports: EvalReport[]): boolean | Promise<boolean>;
+}
+export type ReporterDef<EvalReport> = {
+    name: string;
+} & ReporterBody<EvalReport>;
 export type EvaluatorDef<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
     projectName: string;
     evalName: string;
 } & Evaluator<Input, Output, Expected, Metadata>;
 export type EvaluatorFile = {
-    [evalName: string]: EvaluatorDef<any, any, any, any>;
+    evaluators: {
+        [evalName: string]: {
+            evaluator: EvaluatorDef<any, any, any, any>;
+            reporter?: ReporterDef<unknown> | string;
+        };
+    };
+    reporters: {
+        [reporterName: string]: ReporterDef<unknown>;
+    };
 };
 declare global {
     var _evals: EvaluatorFile;
     var _lazy_load: boolean;
 }
-export declare function Eval<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType>(name: string, evaluator: Evaluator<Input, Output, Expected, Metadata>): Promise<ExperimentSummary>;
+export declare function Eval<Input, Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, EvalReport = boolean>(name: string, evaluator: Evaluator<Input, Output, Expected, Metadata>, reporter?: ReporterDef<EvalReport> | string): Promise<EvalResultWithSummary<Input, Output, Expected, Metadata>>;
+export declare function Reporter<EvalReport>(name: string, reporter: ReporterBody<EvalReport>): ReporterDef<EvalReport>;
 export declare function getLoadedEvals(): EvaluatorFile;
 export interface Filter {
     path: string[];
@@ -89,26 +133,16 @@ export declare function deserializePlainStringAsJSON(s: string): {
     error: unknown;
 };
 export declare function parseFilters(filters: string[]): Filter[];
-export declare function runEvaluator(experiment: Experiment | null, evaluator: EvaluatorDef<any, any, any | void, any | void>, progressReporter: ProgressReporter, filters: Filter[]): Promise<{
-    results: {
-        output: any;
-        metadata: Record<string, unknown>;
-        scores: Record<string, number | null>;
-        error: unknown;
-    }[];
-    summary: ExperimentSummary | null;
-}>;
+export declare function runEvaluator(experiment: Experiment | null, evaluator: EvaluatorDef<any, any, any, any>, progressReporter: ProgressReporter, filters: Filter[]): Promise<EvalResultWithSummary<any, any, any, any>>;
 export declare const error: chalk.Chalk;
 export declare const warning: chalk.Chalk;
 export declare function logError(e: unknown, verbose: boolean): void;
-export declare function reportEvaluatorResult(evaluatorName: string | number, evaluatorResult: {
-    results: {
-        scores: Record<string, number | null>;
-        error: unknown;
-    }[];
-    summary: unknown;
-}, { verbose, jsonl, }: {
-    verbose: boolean;
-    jsonl: boolean;
-}): void;
+export declare function buildLocalSummary(evaluator: EvaluatorDef<any, any, any, any>, results: EvalResult<any, any, any, any>[]): ExperimentSummary;
+export declare function reportFailures<Input, Output, Expected, Metadata extends BaseMetadata>(evaluator: EvaluatorDef<Input, Output, Expected, Metadata>, failingResults: EvalResult<Input, Output, Expected, Metadata>[], { verbose, jsonl }: ReporterOpts): void;
+/**
+ * The default reporter for Braintrust evaluations. This reporter will log the results
+ * of each evaluation to the console, and will return false (i.e. fail) if any of the
+ * evaluations return an error.
+ */
+export declare const defaultReporter: ReporterDef<boolean>;
 export {};

package/dist/index.d.ts

@@ -43,6 +43,6 @@
  * @module braintrust
  */
 export * from "./logger";
-export { BaseExperiment, Evaluator, EvalTask, Eval, EvalScorerArgs, } from "./framework";
+export { BaseExperiment, Evaluator, EvalTask, Eval, EvalScorerArgs, ReporterBody, Reporter, buildLocalSummary, reportFailures, } from "./framework";
 export * from "./oai";
 export { ParentExperimentIds, ParentProjectLogIds, IdField, InputField, InputsField, OtherExperimentLogFields, ExperimentLogPartialArgs, ExperimentLogFullArgs, LogFeedbackFullArgs, LogCommentFullArgs, CommentEvent, DatasetRecord, } from "@braintrust/core";