braintrust 0.0.97 → 0.0.99

package/dist/framework.d.ts

@@ -1,10 +1,10 @@
 import chalk from "chalk";
-import { Experiment, ExperimentSummary, Metadata, Span, EvalCase } from "./logger";
+import { Experiment, ExperimentSummary, Span, EvalCase, BaseMetadata, DefaultMetadataType } from "./logger";
 import { Score } from "@braintrust/core";
 import { ProgressReporter } from "./progress";
-export type BaseExperiment<Input, Expected> = {
+export type BaseExperiment<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
     _type: "BaseExperiment";
-    _phantom?: [Input, Expected];
+    _phantom?: [Input, Expected, Metadata];
     name?: string;
 };
 /**
@@ -17,24 +17,24 @@ export type BaseExperiment<Input, Expected> = {
  * using your git history (or fall back to timestamps).
  * @returns
  */
-export declare function BaseExperiment<Input = unknown, Expected = unknown>(options?: {
+export declare function BaseExperiment<Input = unknown, Expected = unknown, Metadata extends BaseMetadata = DefaultMetadataType>(options?: {
     name?: string;
-}): BaseExperiment<Input, Expected>;
-export type EvalData<Input, Expected> = EvalCase<Input, Expected>[] | (() => EvalCase<Input, Expected>[]) | (() => Promise<EvalCase<Input, Expected>[]>) | AsyncGenerator<EvalCase<Input, Expected>> | BaseExperiment<Input, Expected> | (() => BaseExperiment<Input, Expected>);
+}): BaseExperiment<Input, Expected, Metadata>;
+export type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
 export type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
 export interface EvalHooks {
     meta: (info: Record<string, unknown>) => void;
     span: Span;
 }
-export type EvalScorerArgs<Input, Output, Expected> = EvalCase<Input, Expected> & {
+export type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;
 };
-export type EvalScorer<Input, Output, Expected> = (args: EvalScorerArgs<Input, Output, Expected>) => Score | Promise<Score>;
-export interface Evaluator<Input, Output, Expected> {
+export type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => Score | Promise<Score>;
+export interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> {
     /**
      * A function that returns a list of inputs, expected outputs, and metadata.
      */
-    data: EvalData<Input, Expected>;
+    data: EvalData<Input, Expected, Metadata>;
     /**
      * A function that takes an input and returns an output.
      */
@@ -42,7 +42,7 @@ export interface Evaluator<Input, Output, Expected> {
     /**
      * A set of functions that take an input, output, and expected value and return a score.
      */
-    scores: EvalScorer<Input, Output, Expected>[];
+    scores: EvalScorer<Input, Output, Expected, Metadata>[];
     /**
      * An optional name for the experiment.
      */
@@ -56,24 +56,24 @@ export interface Evaluator<Input, Output, Expected> {
     /**
      * Optional additional metadata for the experiment.
      */
-    metadata?: Metadata;
+    metadata?: Record<string, unknown>;
     /**
      * Whether the experiment should be public. Defaults to false.
      */
     isPublic?: boolean;
 }
-export type EvaluatorDef<Input, Output, Expected> = {
+export type EvaluatorDef<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
     projectName: string;
     evalName: string;
-} & Evaluator<Input, Output, Expected>;
+} & Evaluator<Input, Output, Expected, Metadata>;
 export type EvaluatorFile = {
-    [evalName: string]: EvaluatorDef<any, any, any>;
+    [evalName: string]: EvaluatorDef<any, any, any, any>;
 };
 declare global {
     var _evals: EvaluatorFile;
     var _lazy_load: boolean;
 }
-export declare function Eval<Input, Output, Expected>(name: string, evaluator: Evaluator<Input, Output, Expected>): Promise<ExperimentSummary>;
+export declare function Eval<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType>(name: string, evaluator: Evaluator<Input, Output, Expected, Metadata>): Promise<ExperimentSummary>;
 export declare function getLoadedEvals(): EvaluatorFile;
 export interface Filter {
     path: string[];
@@ -88,10 +88,10 @@ export declare function deserializePlainStringAsJSON(s: string): {
     error: unknown;
 };
 export declare function parseFilters(filters: string[]): Filter[];
-export declare function runEvaluator(experiment: Experiment | null, evaluator: EvaluatorDef<unknown, unknown, unknown>, progressReporter: ProgressReporter, filters: Filter[]): Promise<{
+export declare function runEvaluator(experiment: Experiment | null, evaluator: EvaluatorDef<any, any, any | void, any | void>, progressReporter: ProgressReporter, filters: Filter[]): Promise<{
     results: {
         output: any;
-        metadata: Metadata;
+        metadata: Record<string, unknown>;
         scores: Record<string, number | null>;
         error: unknown;
     }[];

package/dist/gitutil.d.ts

@@ -1,7 +1,7 @@
-import { GitMetadataSettings, RepoStatus } from "@braintrust/core";
+import { GitMetadataSettings, RepoInfo } from "@braintrust/core";
 /**
  * Information about the current HEAD of the repo.
  */
 export declare function currentRepo(): Promise<import("simple-git").SimpleGit | null>;
 export declare function getPastNAncestors(n?: number, remote?: string | undefined): Promise<string[]>;
-export declare function getRepoStatus(settings?: GitMetadataSettings): Promise<RepoStatus | undefined>;
+export declare function getRepoInfo(settings?: GitMetadataSettings): Promise<RepoInfo | undefined>;

package/dist/index.d.ts

@@ -45,3 +45,4 @@
 export * from "./logger";
 export { BaseExperiment, Evaluator, EvalTask, Eval, EvalScorerArgs, } from "./framework";
 export * from "./oai";
+export { ParentExperimentIds, ParentProjectLogIds, IdField, InputField, InputsField, OtherExperimentLogFields, ExperimentLogPartialArgs, ExperimentLogFullArgs, LogFeedbackFullArgs, LogCommentFullArgs, CommentEvent, DatasetRecord, } from "@braintrust/core";