braintrust 0.2.6 → 0.3.7

package/dev/dist/index.d.mts

@@ -1,5 +1,216 @@
-import { TRANSACTION_ID_FIELD, TransactionId, ExperimentEvent, DEFAULT_IS_LEGACY_DATASET, DatasetRecord, ExperimentLogFullArgs, ExperimentLogPartialArgs, LogFeedbackFullArgs, SpanType, IdField, BackgroundLogEvent, Score } from '@braintrust/core';
-import { z } from 'zod';
+import { z } from 'zod/v3';
+import { z as z$1 } from 'zod';
+
+declare const TRANSACTION_ID_FIELD = "_xact_id";
+declare const IS_MERGE_FIELD = "_is_merge";
+declare const MERGE_PATHS_FIELD = "_merge_paths";
+declare const AUDIT_SOURCE_FIELD = "_audit_source";
+declare const AUDIT_METADATA_FIELD = "_audit_metadata";
+declare const VALID_SOURCES: readonly ["app", "api", "external"];
+type Source = (typeof VALID_SOURCES)[number];
+declare const ASYNC_SCORING_CONTROL_FIELD = "_async_scoring_control";
+declare const SKIP_ASYNC_SCORING_FIELD = "_skip_async_scoring";
+type TransactionId = string;
+
+declare const AsyncScoringControl: z.ZodUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"score_update">;
+    token: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    token: string;
+    kind: "score_update";
+}, {
+    token: string;
+    kind: "score_update";
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"state_override">;
+    state: z.ZodUnion<[z.ZodObject<{
+        status: z.ZodLiteral<"enabled">;
+        token: z.ZodString;
+        function_ids: z.ZodArray<z.ZodUnknown, "many">;
+        skip_logging: z.ZodOptional<z.ZodUnion<[z.ZodBoolean, z.ZodNull]>>;
+    }, "strip", z.ZodTypeAny, {
+        status: "enabled";
+        token: string;
+        function_ids: unknown[];
+        skip_logging?: boolean | null | undefined;
+    }, {
+        status: "enabled";
+        token: string;
+        function_ids: unknown[];
+        skip_logging?: boolean | null | undefined;
+    }>, z.ZodObject<{
+        status: z.ZodLiteral<"disabled">;
+    }, "strip", z.ZodTypeAny, {
+        status: "disabled";
+    }, {
+        status: "disabled";
+    }>, z.ZodNull, z.ZodNull]>;
+}, "strip", z.ZodTypeAny, {
+    kind: "state_override";
+    state: {
+        status: "enabled";
+        token: string;
+        function_ids: unknown[];
+        skip_logging?: boolean | null | undefined;
+    } | {
+        status: "disabled";
+    } | null;
+}, {
+    kind: "state_override";
+    state: {
+        status: "enabled";
+        token: string;
+        function_ids: unknown[];
+        skip_logging?: boolean | null | undefined;
+    } | {
+        status: "disabled";
+    } | null;
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"state_force_reselect">;
+}, "strip", z.ZodTypeAny, {
+    kind: "state_force_reselect";
+}, {
+    kind: "state_force_reselect";
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"state_enabled_force_rescore">;
+}, "strip", z.ZodTypeAny, {
+    kind: "state_enabled_force_rescore";
+}, {
+    kind: "state_enabled_force_rescore";
+}>]>;
+type AsyncScoringControlType = z.infer<typeof AsyncScoringControl>;
+declare const ObjectReference$1: z.ZodObject<{
+    object_type: z.ZodEnum<["project_logs", "experiment", "dataset", "prompt", "function", "prompt_session"]>;
+    object_id: z.ZodString;
+    id: z.ZodString;
+    _xact_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNull]>>;
+    created: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNull]>>;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
+    object_id: string;
+    created?: string | null | undefined;
+    _xact_id?: string | null | undefined;
+}, {
+    id: string;
+    object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
+    object_id: string;
+    created?: string | null | undefined;
+    _xact_id?: string | null | undefined;
+}>;
+
+type IdField = {
+    id: string;
+};
+type InputField = {
+    input: unknown;
+};
+type OtherExperimentLogFields = {
+    output: unknown;
+    expected: unknown;
+    error: unknown;
+    tags: string[];
+    scores: Record<string, number | null>;
+    metadata: Record<string, unknown>;
+    metrics: Record<string, unknown>;
+    datasetRecordId: string;
+    origin: z.infer<typeof ObjectReference$1>;
+    span_attributes: Record<string, unknown>;
+    [ASYNC_SCORING_CONTROL_FIELD]: AsyncScoringControlType;
+    [MERGE_PATHS_FIELD]: string[][];
+    [SKIP_ASYNC_SCORING_FIELD]: boolean;
+};
+type ExperimentLogPartialArgs = Partial<OtherExperimentLogFields> & Partial<InputField>;
+type ExperimentLogFullArgs = Partial<Omit<OtherExperimentLogFields, "output" | "scores">> & Required<Pick<OtherExperimentLogFields, "output" | "scores">> & Partial<InputField> & Partial<IdField>;
+type LogFeedbackFullArgs = IdField & Partial<Omit<OtherExperimentLogFields, "output" | "metrics" | "datasetRecordId"> & {
+    comment: string;
+    source: Source;
+}>;
+interface ParentExperimentIds {
+    experiment_id: string;
+}
+interface ParentProjectLogIds {
+    project_id: string;
+    log_id: "g";
+}
+interface ParentPlaygroundLogIds {
+    prompt_session_id: string;
+    log_id: "x";
+}
+type ExperimentEvent = Partial<InputField> & Partial<OtherExperimentLogFields> & {
+    id: string;
+    span_id?: string;
+    root_span_id?: string;
+    experiment_id: string;
+    [IS_MERGE_FIELD]: boolean;
+} & Partial<{
+    created: string;
+    span_parents: string[];
+    span_attributes: Record<string, unknown>;
+    context: Record<string, unknown>;
+    [AUDIT_SOURCE_FIELD]: Source;
+    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
+}>;
+type DatasetEvent = {
+    input?: unknown;
+    tags?: string[];
+    metadata?: unknown;
+    created?: string;
+    id: string;
+    dataset_id: string;
+} & ({
+    expected?: unknown;
+} | {
+    output?: unknown;
+});
+type LoggingEvent = Omit<ExperimentEvent, "experiment_id"> & {
+    project_id: string;
+    log_id: "g";
+};
+type PlaygroundLogEvent = Omit<ExperimentEvent, "experiment_id"> & {
+    prompt_session_id: string;
+    log_id: "x";
+};
+type CommentEvent = IdField & {
+    created: string;
+    origin: {
+        id: string;
+    };
+    comment: {
+        text: string;
+    };
+    [AUDIT_SOURCE_FIELD]: Source;
+    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
+} & (ParentExperimentIds | ParentProjectLogIds | ParentPlaygroundLogIds);
+type BackgroundLogEvent = ExperimentEvent | DatasetEvent | LoggingEvent | PlaygroundLogEvent | CommentEvent;
+declare const DEFAULT_IS_LEGACY_DATASET = false;
+interface LegacyDatasetRecord {
+    id: string;
+    input: any;
+    output: any;
+    metadata: any;
+}
+interface NewDatasetRecord {
+    id: string;
+    input: any;
+    expected: any;
+    tags: any;
+    metadata: any;
+}
+type DatasetRecord<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET> = IsLegacyDataset extends true ? LegacyDatasetRecord : NewDatasetRecord;
+
+interface Score {
+    name: string;
+    score: number | null;
+    metadata?: Record<string, unknown>;
+    /**
+     * @deprecated
+     */
+    error?: unknown;
+}
+
+declare const spanTypeAttributeValues: readonly ["llm", "score", "function", "eval", "task", "tool"];
+type SpanType = (typeof spanTypeAttributeValues)[number];
 
 declare const AnyModelParams: z.ZodObject<{
     temperature: z.ZodOptional<z.ZodNumber>;
@@ -7251,25 +7462,25 @@ declare const NOOP_SPAN: NoopSpan;
 declare global {
     var __inherited_braintrust_state: BraintrustState;
 }
-declare const loginSchema: z.ZodObject<{
-    appUrl: z.ZodString;
-    appPublicUrl: z.ZodString;
-    orgName: z.ZodString;
-    apiUrl: z.ZodString;
-    proxyUrl: z.ZodString;
-    loginToken: z.ZodString;
-    orgId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
-    gitMetadataSettings: z.ZodOptional<z.ZodNullable<z.ZodObject<{
-        collect: z.ZodEnum<["all", "none", "some"]>;
-        fields: z.ZodOptional<z.ZodArray<z.ZodEnum<["commit", "branch", "tag", "dirty", "author_name", "author_email", "commit_message", "commit_time", "git_diff"]>, "many">>;
-    }, "strip", z.ZodTypeAny, {
+declare const loginSchema: z$1.ZodObject<{
+    appUrl: z$1.ZodString;
+    appPublicUrl: z$1.ZodString;
+    orgName: z$1.ZodString;
+    apiUrl: z$1.ZodString;
+    proxyUrl: z$1.ZodString;
+    loginToken: z$1.ZodString;
+    orgId: z$1.ZodOptional<z$1.ZodNullable<z$1.ZodString>>;
+    gitMetadataSettings: z$1.ZodOptional<z$1.ZodNullable<z$1.ZodObject<{
+        collect: z$1.ZodEnum<["all", "none", "some"]>;
+        fields: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<["commit", "branch", "tag", "dirty", "author_name", "author_email", "commit_message", "commit_time", "git_diff"]>, "many">>;
+    }, "strip", z$1.ZodTypeAny, {
         collect: "some" | "none" | "all";
         fields?: ("dirty" | "commit" | "branch" | "tag" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     }, {
         collect: "some" | "none" | "all";
         fields?: ("dirty" | "commit" | "branch" | "tag" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     }>>>;
-}, "strict", z.ZodTypeAny, {
+}, "strict", z$1.ZodTypeAny, {
     appUrl: string;
     appPublicUrl: string;
     orgName: string;
@@ -7294,7 +7505,7 @@ declare const loginSchema: z.ZodObject<{
         fields?: ("dirty" | "commit" | "branch" | "tag" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     } | null | undefined;
 }>;
-type SerializedBraintrustState = z.infer<typeof loginSchema>;
+type SerializedBraintrustState = z$1.infer<typeof loginSchema>;
 declare class BraintrustState {
     private loginParams;
     id: string;
@@ -10968,6 +11179,7 @@ declare global {
 interface DevServerOpts {
     host: string;
     port: number;
+    orgName?: string;
 }
 declare function runDevServer(evaluators: EvaluatorDef<any, any, any, any, any>[], opts: DevServerOpts): void;
 

package/dev/dist/index.d.ts

@@ -1,5 +1,216 @@
-import { TRANSACTION_ID_FIELD, TransactionId, ExperimentEvent, DEFAULT_IS_LEGACY_DATASET, DatasetRecord, ExperimentLogFullArgs, ExperimentLogPartialArgs, LogFeedbackFullArgs, SpanType, IdField, BackgroundLogEvent, Score } from '@braintrust/core';
-import { z } from 'zod';
+import { z } from 'zod/v3';
+import { z as z$1 } from 'zod';
+
+declare const TRANSACTION_ID_FIELD = "_xact_id";
+declare const IS_MERGE_FIELD = "_is_merge";
+declare const MERGE_PATHS_FIELD = "_merge_paths";
+declare const AUDIT_SOURCE_FIELD = "_audit_source";
+declare const AUDIT_METADATA_FIELD = "_audit_metadata";
+declare const VALID_SOURCES: readonly ["app", "api", "external"];
+type Source = (typeof VALID_SOURCES)[number];
+declare const ASYNC_SCORING_CONTROL_FIELD = "_async_scoring_control";
+declare const SKIP_ASYNC_SCORING_FIELD = "_skip_async_scoring";
+type TransactionId = string;
+
+declare const AsyncScoringControl: z.ZodUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"score_update">;
+    token: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    token: string;
+    kind: "score_update";
+}, {
+    token: string;
+    kind: "score_update";
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"state_override">;
+    state: z.ZodUnion<[z.ZodObject<{
+        status: z.ZodLiteral<"enabled">;
+        token: z.ZodString;
+        function_ids: z.ZodArray<z.ZodUnknown, "many">;
+        skip_logging: z.ZodOptional<z.ZodUnion<[z.ZodBoolean, z.ZodNull]>>;
+    }, "strip", z.ZodTypeAny, {
+        status: "enabled";
+        token: string;
+        function_ids: unknown[];
+        skip_logging?: boolean | null | undefined;
+    }, {
+        status: "enabled";
+        token: string;
+        function_ids: unknown[];
+        skip_logging?: boolean | null | undefined;
+    }>, z.ZodObject<{
+        status: z.ZodLiteral<"disabled">;
+    }, "strip", z.ZodTypeAny, {
+        status: "disabled";
+    }, {
+        status: "disabled";
+    }>, z.ZodNull, z.ZodNull]>;
+}, "strip", z.ZodTypeAny, {
+    kind: "state_override";
+    state: {
+        status: "enabled";
+        token: string;
+        function_ids: unknown[];
+        skip_logging?: boolean | null | undefined;
+    } | {
+        status: "disabled";
+    } | null;
+}, {
+    kind: "state_override";
+    state: {
+        status: "enabled";
+        token: string;
+        function_ids: unknown[];
+        skip_logging?: boolean | null | undefined;
+    } | {
+        status: "disabled";
+    } | null;
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"state_force_reselect">;
+}, "strip", z.ZodTypeAny, {
+    kind: "state_force_reselect";
+}, {
+    kind: "state_force_reselect";
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"state_enabled_force_rescore">;
+}, "strip", z.ZodTypeAny, {
+    kind: "state_enabled_force_rescore";
+}, {
+    kind: "state_enabled_force_rescore";
+}>]>;
+type AsyncScoringControlType = z.infer<typeof AsyncScoringControl>;
+declare const ObjectReference$1: z.ZodObject<{
+    object_type: z.ZodEnum<["project_logs", "experiment", "dataset", "prompt", "function", "prompt_session"]>;
+    object_id: z.ZodString;
+    id: z.ZodString;
+    _xact_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNull]>>;
+    created: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNull]>>;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
+    object_id: string;
+    created?: string | null | undefined;
+    _xact_id?: string | null | undefined;
+}, {
+    id: string;
+    object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
+    object_id: string;
+    created?: string | null | undefined;
+    _xact_id?: string | null | undefined;
+}>;
+
+type IdField = {
+    id: string;
+};
+type InputField = {
+    input: unknown;
+};
+type OtherExperimentLogFields = {
+    output: unknown;
+    expected: unknown;
+    error: unknown;
+    tags: string[];
+    scores: Record<string, number | null>;
+    metadata: Record<string, unknown>;
+    metrics: Record<string, unknown>;
+    datasetRecordId: string;
+    origin: z.infer<typeof ObjectReference$1>;
+    span_attributes: Record<string, unknown>;
+    [ASYNC_SCORING_CONTROL_FIELD]: AsyncScoringControlType;
+    [MERGE_PATHS_FIELD]: string[][];
+    [SKIP_ASYNC_SCORING_FIELD]: boolean;
+};
+type ExperimentLogPartialArgs = Partial<OtherExperimentLogFields> & Partial<InputField>;
+type ExperimentLogFullArgs = Partial<Omit<OtherExperimentLogFields, "output" | "scores">> & Required<Pick<OtherExperimentLogFields, "output" | "scores">> & Partial<InputField> & Partial<IdField>;
+type LogFeedbackFullArgs = IdField & Partial<Omit<OtherExperimentLogFields, "output" | "metrics" | "datasetRecordId"> & {
+    comment: string;
+    source: Source;
+}>;
+interface ParentExperimentIds {
+    experiment_id: string;
+}
+interface ParentProjectLogIds {
+    project_id: string;
+    log_id: "g";
+}
+interface ParentPlaygroundLogIds {
+    prompt_session_id: string;
+    log_id: "x";
+}
+type ExperimentEvent = Partial<InputField> & Partial<OtherExperimentLogFields> & {
+    id: string;
+    span_id?: string;
+    root_span_id?: string;
+    experiment_id: string;
+    [IS_MERGE_FIELD]: boolean;
+} & Partial<{
+    created: string;
+    span_parents: string[];
+    span_attributes: Record<string, unknown>;
+    context: Record<string, unknown>;
+    [AUDIT_SOURCE_FIELD]: Source;
+    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
+}>;
+type DatasetEvent = {
+    input?: unknown;
+    tags?: string[];
+    metadata?: unknown;
+    created?: string;
+    id: string;
+    dataset_id: string;
+} & ({
+    expected?: unknown;
+} | {
+    output?: unknown;
+});
+type LoggingEvent = Omit<ExperimentEvent, "experiment_id"> & {
+    project_id: string;
+    log_id: "g";
+};
+type PlaygroundLogEvent = Omit<ExperimentEvent, "experiment_id"> & {
+    prompt_session_id: string;
+    log_id: "x";
+};
+type CommentEvent = IdField & {
+    created: string;
+    origin: {
+        id: string;
+    };
+    comment: {
+        text: string;
+    };
+    [AUDIT_SOURCE_FIELD]: Source;
+    [AUDIT_METADATA_FIELD]?: Record<string, unknown>;
+} & (ParentExperimentIds | ParentProjectLogIds | ParentPlaygroundLogIds);
+type BackgroundLogEvent = ExperimentEvent | DatasetEvent | LoggingEvent | PlaygroundLogEvent | CommentEvent;
+declare const DEFAULT_IS_LEGACY_DATASET = false;
+interface LegacyDatasetRecord {
+    id: string;
+    input: any;
+    output: any;
+    metadata: any;
+}
+interface NewDatasetRecord {
+    id: string;
+    input: any;
+    expected: any;
+    tags: any;
+    metadata: any;
+}
+type DatasetRecord<IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET> = IsLegacyDataset extends true ? LegacyDatasetRecord : NewDatasetRecord;
+
+interface Score {
+    name: string;
+    score: number | null;
+    metadata?: Record<string, unknown>;
+    /**
+     * @deprecated
+     */
+    error?: unknown;
+}
+
+declare const spanTypeAttributeValues: readonly ["llm", "score", "function", "eval", "task", "tool"];
+type SpanType = (typeof spanTypeAttributeValues)[number];
 
 declare const AnyModelParams: z.ZodObject<{
     temperature: z.ZodOptional<z.ZodNumber>;
@@ -7251,25 +7462,25 @@ declare const NOOP_SPAN: NoopSpan;
 declare global {
     var __inherited_braintrust_state: BraintrustState;
 }
-declare const loginSchema: z.ZodObject<{
-    appUrl: z.ZodString;
-    appPublicUrl: z.ZodString;
-    orgName: z.ZodString;
-    apiUrl: z.ZodString;
-    proxyUrl: z.ZodString;
-    loginToken: z.ZodString;
-    orgId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
-    gitMetadataSettings: z.ZodOptional<z.ZodNullable<z.ZodObject<{
-        collect: z.ZodEnum<["all", "none", "some"]>;
-        fields: z.ZodOptional<z.ZodArray<z.ZodEnum<["commit", "branch", "tag", "dirty", "author_name", "author_email", "commit_message", "commit_time", "git_diff"]>, "many">>;
-    }, "strip", z.ZodTypeAny, {
+declare const loginSchema: z$1.ZodObject<{
+    appUrl: z$1.ZodString;
+    appPublicUrl: z$1.ZodString;
+    orgName: z$1.ZodString;
+    apiUrl: z$1.ZodString;
+    proxyUrl: z$1.ZodString;
+    loginToken: z$1.ZodString;
+    orgId: z$1.ZodOptional<z$1.ZodNullable<z$1.ZodString>>;
+    gitMetadataSettings: z$1.ZodOptional<z$1.ZodNullable<z$1.ZodObject<{
+        collect: z$1.ZodEnum<["all", "none", "some"]>;
+        fields: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<["commit", "branch", "tag", "dirty", "author_name", "author_email", "commit_message", "commit_time", "git_diff"]>, "many">>;
+    }, "strip", z$1.ZodTypeAny, {
         collect: "some" | "none" | "all";
         fields?: ("dirty" | "commit" | "branch" | "tag" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     }, {
         collect: "some" | "none" | "all";
         fields?: ("dirty" | "commit" | "branch" | "tag" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     }>>>;
-}, "strict", z.ZodTypeAny, {
+}, "strict", z$1.ZodTypeAny, {
     appUrl: string;
     appPublicUrl: string;
     orgName: string;
@@ -7294,7 +7505,7 @@ declare const loginSchema: z.ZodObject<{
         fields?: ("dirty" | "commit" | "branch" | "tag" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     } | null | undefined;
 }>;
-type SerializedBraintrustState = z.infer<typeof loginSchema>;
+type SerializedBraintrustState = z$1.infer<typeof loginSchema>;
 declare class BraintrustState {
     private loginParams;
     id: string;
@@ -10968,6 +11179,7 @@ declare global {
 interface DevServerOpts {
     host: string;
     port: number;
+    orgName?: string;
 }
 declare function runDevServer(evaluators: EvaluatorDef<any, any, any, any, any>[], opts: DevServerOpts): void;