braintrust 0.0.201 → 0.0.202

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. package/dist/browser.d.mts +26 -24
  2. package/dist/browser.d.ts +26 -24
  3. package/dist/browser.js +21 -11
  4. package/dist/browser.mjs +21 -11
  5. package/dist/cli.js +77 -67
  6. package/dist/index.d.mts +30 -28
  7. package/dist/index.d.ts +30 -28
  8. package/dist/index.js +40 -16
  9. package/dist/index.mjs +40 -16
  10. package/examples/anthropic_beta_example.ts +59 -0
  11. package/examples/anthropic_example.ts +0 -1
  12. package/package.json +1 -1
package/dist/browser.d.mts CHANGED
@@ -176,6 +176,8 @@ declare class LazyValue<T> {
176
176
  get hasSucceeded(): boolean;
177
177
  }
178
178
 
179
+ /// <reference lib="dom" />
180
+
179
181
  type SetCurrentArg = {
180
182
  setCurrent?: boolean;
181
183
  };
@@ -1640,29 +1642,29 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1640
1642
  type: z.ZodLiteral<"text_delta">;
1641
1643
  data: z.ZodString;
1642
1644
  }, "strip", z.ZodTypeAny, {
1643
- data: string;
1644
1645
  type: "text_delta";
1645
- }, {
1646
1646
  data: string;
1647
+ }, {
1647
1648
  type: "text_delta";
1649
+ data: string;
1648
1650
  }>, z.ZodObject<{
1649
1651
  type: z.ZodLiteral<"json_delta">;
1650
1652
  data: z.ZodString;
1651
1653
  }, "strip", z.ZodTypeAny, {
1652
- data: string;
1653
1654
  type: "json_delta";
1654
- }, {
1655
1655
  data: string;
1656
+ }, {
1656
1657
  type: "json_delta";
1658
+ data: string;
1657
1659
  }>, z.ZodObject<{
1658
1660
  type: z.ZodLiteral<"error">;
1659
1661
  data: z.ZodString;
1660
1662
  }, "strip", z.ZodTypeAny, {
1661
- data: string;
1662
1663
  type: "error";
1663
- }, {
1664
1664
  data: string;
1665
+ }, {
1665
1666
  type: "error";
1667
+ data: string;
1666
1668
  }>, z.ZodObject<{
1667
1669
  type: z.ZodLiteral<"console">;
1668
1670
  data: z.ZodObject<{
@@ -1676,17 +1678,17 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1676
1678
  stream: "stderr" | "stdout";
1677
1679
  }>;
1678
1680
  }, "strip", z.ZodTypeAny, {
1681
+ type: "console";
1679
1682
  data: {
1680
1683
  message: string;
1681
1684
  stream: "stderr" | "stdout";
1682
1685
  };
1683
- type: "console";
1684
1686
  }, {
1687
+ type: "console";
1685
1688
  data: {
1686
1689
  message: string;
1687
1690
  stream: "stderr" | "stdout";
1688
1691
  };
1689
- type: "console";
1690
1692
  }>, z.ZodObject<{
1691
1693
  type: z.ZodLiteral<"progress">;
1692
1694
  data: z.ZodObject<{
@@ -1699,13 +1701,13 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1699
1701
  _xact_id: z.ZodString;
1700
1702
  created: z.ZodOptional<z.ZodNullable<z.ZodString>>;
1701
1703
  }, "strip", z.ZodTypeAny, {
1702
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1704
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1703
1705
  object_id: string;
1704
1706
  id: string;
1705
1707
  _xact_id: string;
1706
1708
  created?: string | null | undefined;
1707
1709
  }, {
1708
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1710
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1709
1711
  object_id: string;
1710
1712
  id: string;
1711
1713
  _xact_id: string;
@@ -1722,10 +1724,10 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1722
1724
  name: string;
1723
1725
  object_type: "prompt" | "tool" | "scorer" | "task" | "agent";
1724
1726
  id: string;
1725
- event: "error" | "text_delta" | "json_delta" | "progress" | "console" | "start" | "done";
1727
+ event: "text_delta" | "json_delta" | "error" | "console" | "progress" | "start" | "done";
1726
1728
  output_type: "completion" | "score" | "any";
1727
1729
  origin?: {
1728
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1730
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1729
1731
  object_id: string;
1730
1732
  id: string;
1731
1733
  _xact_id: string;
@@ -1737,10 +1739,10 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1737
1739
  name: string;
1738
1740
  object_type: "prompt" | "tool" | "scorer" | "task" | "agent";
1739
1741
  id: string;
1740
- event: "error" | "text_delta" | "json_delta" | "progress" | "console" | "start" | "done";
1742
+ event: "text_delta" | "json_delta" | "error" | "console" | "progress" | "start" | "done";
1741
1743
  output_type: "completion" | "score" | "any";
1742
1744
  origin?: {
1743
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1745
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1744
1746
  object_id: string;
1745
1747
  id: string;
1746
1748
  _xact_id: string;
@@ -1748,59 +1750,59 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1748
1750
  } | null | undefined;
1749
1751
  }>;
1750
1752
  }, "strip", z.ZodTypeAny, {
1753
+ type: "progress";
1751
1754
  data: {
1752
1755
  format: "code" | "global" | "graph" | "llm";
1753
1756
  data: string;
1754
1757
  name: string;
1755
1758
  object_type: "prompt" | "tool" | "scorer" | "task" | "agent";
1756
1759
  id: string;
1757
- event: "error" | "text_delta" | "json_delta" | "progress" | "console" | "start" | "done";
1760
+ event: "text_delta" | "json_delta" | "error" | "console" | "progress" | "start" | "done";
1758
1761
  output_type: "completion" | "score" | "any";
1759
1762
  origin?: {
1760
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1763
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1761
1764
  object_id: string;
1762
1765
  id: string;
1763
1766
  _xact_id: string;
1764
1767
  created?: string | null | undefined;
1765
1768
  } | null | undefined;
1766
1769
  };
1767
- type: "progress";
1768
1770
  }, {
1771
+ type: "progress";
1769
1772
  data: {
1770
1773
  format: "code" | "global" | "graph" | "llm";
1771
1774
  data: string;
1772
1775
  name: string;
1773
1776
  object_type: "prompt" | "tool" | "scorer" | "task" | "agent";
1774
1777
  id: string;
1775
- event: "error" | "text_delta" | "json_delta" | "progress" | "console" | "start" | "done";
1778
+ event: "text_delta" | "json_delta" | "error" | "console" | "progress" | "start" | "done";
1776
1779
  output_type: "completion" | "score" | "any";
1777
1780
  origin?: {
1778
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1781
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1779
1782
  object_id: string;
1780
1783
  id: string;
1781
1784
  _xact_id: string;
1782
1785
  created?: string | null | undefined;
1783
1786
  } | null | undefined;
1784
1787
  };
1785
- type: "progress";
1786
1788
  }>, z.ZodObject<{
1787
1789
  type: z.ZodLiteral<"start">;
1788
1790
  data: z.ZodString;
1789
1791
  }, "strip", z.ZodTypeAny, {
1790
- data: string;
1791
1792
  type: "start";
1792
- }, {
1793
1793
  data: string;
1794
+ }, {
1794
1795
  type: "start";
1796
+ data: string;
1795
1797
  }>, z.ZodObject<{
1796
1798
  type: z.ZodLiteral<"done">;
1797
1799
  data: z.ZodString;
1798
1800
  }, "strip", z.ZodTypeAny, {
1799
- data: string;
1800
1801
  type: "done";
1801
- }, {
1802
1802
  data: string;
1803
+ }, {
1803
1804
  type: "done";
1805
+ data: string;
1804
1806
  }>]>;
1805
1807
  /**
1806
1808
  * A chunk of data from a Braintrust stream. Each chunk type matches
package/dist/browser.d.ts CHANGED
@@ -176,6 +176,8 @@ declare class LazyValue<T> {
176
176
  get hasSucceeded(): boolean;
177
177
  }
178
178
 
179
+ /// <reference lib="dom" />
180
+
179
181
  type SetCurrentArg = {
180
182
  setCurrent?: boolean;
181
183
  };
@@ -1640,29 +1642,29 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1640
1642
  type: z.ZodLiteral<"text_delta">;
1641
1643
  data: z.ZodString;
1642
1644
  }, "strip", z.ZodTypeAny, {
1643
- data: string;
1644
1645
  type: "text_delta";
1645
- }, {
1646
1646
  data: string;
1647
+ }, {
1647
1648
  type: "text_delta";
1649
+ data: string;
1648
1650
  }>, z.ZodObject<{
1649
1651
  type: z.ZodLiteral<"json_delta">;
1650
1652
  data: z.ZodString;
1651
1653
  }, "strip", z.ZodTypeAny, {
1652
- data: string;
1653
1654
  type: "json_delta";
1654
- }, {
1655
1655
  data: string;
1656
+ }, {
1656
1657
  type: "json_delta";
1658
+ data: string;
1657
1659
  }>, z.ZodObject<{
1658
1660
  type: z.ZodLiteral<"error">;
1659
1661
  data: z.ZodString;
1660
1662
  }, "strip", z.ZodTypeAny, {
1661
- data: string;
1662
1663
  type: "error";
1663
- }, {
1664
1664
  data: string;
1665
+ }, {
1665
1666
  type: "error";
1667
+ data: string;
1666
1668
  }>, z.ZodObject<{
1667
1669
  type: z.ZodLiteral<"console">;
1668
1670
  data: z.ZodObject<{
@@ -1676,17 +1678,17 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1676
1678
  stream: "stderr" | "stdout";
1677
1679
  }>;
1678
1680
  }, "strip", z.ZodTypeAny, {
1681
+ type: "console";
1679
1682
  data: {
1680
1683
  message: string;
1681
1684
  stream: "stderr" | "stdout";
1682
1685
  };
1683
- type: "console";
1684
1686
  }, {
1687
+ type: "console";
1685
1688
  data: {
1686
1689
  message: string;
1687
1690
  stream: "stderr" | "stdout";
1688
1691
  };
1689
- type: "console";
1690
1692
  }>, z.ZodObject<{
1691
1693
  type: z.ZodLiteral<"progress">;
1692
1694
  data: z.ZodObject<{
@@ -1699,13 +1701,13 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1699
1701
  _xact_id: z.ZodString;
1700
1702
  created: z.ZodOptional<z.ZodNullable<z.ZodString>>;
1701
1703
  }, "strip", z.ZodTypeAny, {
1702
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1704
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1703
1705
  object_id: string;
1704
1706
  id: string;
1705
1707
  _xact_id: string;
1706
1708
  created?: string | null | undefined;
1707
1709
  }, {
1708
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1710
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1709
1711
  object_id: string;
1710
1712
  id: string;
1711
1713
  _xact_id: string;
@@ -1722,10 +1724,10 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1722
1724
  name: string;
1723
1725
  object_type: "prompt" | "tool" | "scorer" | "task" | "agent";
1724
1726
  id: string;
1725
- event: "error" | "text_delta" | "json_delta" | "progress" | "console" | "start" | "done";
1727
+ event: "text_delta" | "json_delta" | "error" | "console" | "progress" | "start" | "done";
1726
1728
  output_type: "completion" | "score" | "any";
1727
1729
  origin?: {
1728
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1730
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1729
1731
  object_id: string;
1730
1732
  id: string;
1731
1733
  _xact_id: string;
@@ -1737,10 +1739,10 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1737
1739
  name: string;
1738
1740
  object_type: "prompt" | "tool" | "scorer" | "task" | "agent";
1739
1741
  id: string;
1740
- event: "error" | "text_delta" | "json_delta" | "progress" | "console" | "start" | "done";
1742
+ event: "text_delta" | "json_delta" | "error" | "console" | "progress" | "start" | "done";
1741
1743
  output_type: "completion" | "score" | "any";
1742
1744
  origin?: {
1743
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1745
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1744
1746
  object_id: string;
1745
1747
  id: string;
1746
1748
  _xact_id: string;
@@ -1748,59 +1750,59 @@ declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
1748
1750
  } | null | undefined;
1749
1751
  }>;
1750
1752
  }, "strip", z.ZodTypeAny, {
1753
+ type: "progress";
1751
1754
  data: {
1752
1755
  format: "code" | "global" | "graph" | "llm";
1753
1756
  data: string;
1754
1757
  name: string;
1755
1758
  object_type: "prompt" | "tool" | "scorer" | "task" | "agent";
1756
1759
  id: string;
1757
- event: "error" | "text_delta" | "json_delta" | "progress" | "console" | "start" | "done";
1760
+ event: "text_delta" | "json_delta" | "error" | "console" | "progress" | "start" | "done";
1758
1761
  output_type: "completion" | "score" | "any";
1759
1762
  origin?: {
1760
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1763
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1761
1764
  object_id: string;
1762
1765
  id: string;
1763
1766
  _xact_id: string;
1764
1767
  created?: string | null | undefined;
1765
1768
  } | null | undefined;
1766
1769
  };
1767
- type: "progress";
1768
1770
  }, {
1771
+ type: "progress";
1769
1772
  data: {
1770
1773
  format: "code" | "global" | "graph" | "llm";
1771
1774
  data: string;
1772
1775
  name: string;
1773
1776
  object_type: "prompt" | "tool" | "scorer" | "task" | "agent";
1774
1777
  id: string;
1775
- event: "error" | "text_delta" | "json_delta" | "progress" | "console" | "start" | "done";
1778
+ event: "text_delta" | "json_delta" | "error" | "console" | "progress" | "start" | "done";
1776
1779
  output_type: "completion" | "score" | "any";
1777
1780
  origin?: {
1778
- object_type: "function" | "experiment" | "dataset" | "prompt" | "prompt_session" | "project_logs";
1781
+ object_type: "function" | "prompt" | "experiment" | "dataset" | "prompt_session" | "project_logs";
1779
1782
  object_id: string;
1780
1783
  id: string;
1781
1784
  _xact_id: string;
1782
1785
  created?: string | null | undefined;
1783
1786
  } | null | undefined;
1784
1787
  };
1785
- type: "progress";
1786
1788
  }>, z.ZodObject<{
1787
1789
  type: z.ZodLiteral<"start">;
1788
1790
  data: z.ZodString;
1789
1791
  }, "strip", z.ZodTypeAny, {
1790
- data: string;
1791
1792
  type: "start";
1792
- }, {
1793
1793
  data: string;
1794
+ }, {
1794
1795
  type: "start";
1796
+ data: string;
1795
1797
  }>, z.ZodObject<{
1796
1798
  type: z.ZodLiteral<"done">;
1797
1799
  data: z.ZodString;
1798
1800
  }, "strip", z.ZodTypeAny, {
1799
- data: string;
1800
1801
  type: "done";
1801
- }, {
1802
1802
  data: string;
1803
+ }, {
1803
1804
  type: "done";
1805
+ data: string;
1804
1806
  }>]>;
1805
1807
  /**
1806
1808
  * A chunk of data from a Braintrust stream. Each chunk type matches
package/dist/browser.js CHANGED
@@ -3410,16 +3410,26 @@ var Experiment = class extends ObjectFetcher {
3410
3410
  comparisonExperimentName = baseExperiment.name;
3411
3411
  }
3412
3412
  }
3413
- const results = await state.apiConn().get_json(
3414
- "/experiment-comparison2",
3415
- {
3416
- experiment_id: await this.id,
3417
- base_experiment_id: comparisonExperimentId
3418
- },
3419
- 3
3420
- );
3421
- scores = results["scores"];
3422
- metrics = results["metrics"];
3413
+ try {
3414
+ const results = await state.apiConn().get_json(
3415
+ "/experiment-comparison2",
3416
+ {
3417
+ experiment_id: await this.id,
3418
+ base_experiment_id: comparisonExperimentId
3419
+ },
3420
+ 3
3421
+ );
3422
+ scores = results["scores"];
3423
+ metrics = results["metrics"];
3424
+ } catch (e) {
3425
+ console.warn(
3426
+ `Failed to fetch experiment scores and metrics: ${e}
3427
+
3428
+ View complete results in Braintrust or run experiment.summarize() again.`
3429
+ );
3430
+ scores = {};
3431
+ metrics = {};
3432
+ }
3423
3433
  }
3424
3434
  return {
3425
3435
  projectName: (await this.project).name,
@@ -3430,7 +3440,7 @@ var Experiment = class extends ObjectFetcher {
3430
3440
  experimentUrl,
3431
3441
  comparisonExperimentName,
3432
3442
  scores: scores ?? {},
3433
- metrics
3443
+ metrics: metrics ?? {}
3434
3444
  };
3435
3445
  }
3436
3446
  /**
package/dist/browser.mjs CHANGED
@@ -3353,16 +3353,26 @@ var Experiment = class extends ObjectFetcher {
3353
3353
  comparisonExperimentName = baseExperiment.name;
3354
3354
  }
3355
3355
  }
3356
- const results = await state.apiConn().get_json(
3357
- "/experiment-comparison2",
3358
- {
3359
- experiment_id: await this.id,
3360
- base_experiment_id: comparisonExperimentId
3361
- },
3362
- 3
3363
- );
3364
- scores = results["scores"];
3365
- metrics = results["metrics"];
3356
+ try {
3357
+ const results = await state.apiConn().get_json(
3358
+ "/experiment-comparison2",
3359
+ {
3360
+ experiment_id: await this.id,
3361
+ base_experiment_id: comparisonExperimentId
3362
+ },
3363
+ 3
3364
+ );
3365
+ scores = results["scores"];
3366
+ metrics = results["metrics"];
3367
+ } catch (e) {
3368
+ console.warn(
3369
+ `Failed to fetch experiment scores and metrics: ${e}
3370
+
3371
+ View complete results in Braintrust or run experiment.summarize() again.`
3372
+ );
3373
+ scores = {};
3374
+ metrics = {};
3375
+ }
3366
3376
  }
3367
3377
  return {
3368
3378
  projectName: (await this.project).name,
@@ -3373,7 +3383,7 @@ var Experiment = class extends ObjectFetcher {
3373
3383
  experimentUrl,
3374
3384
  comparisonExperimentName,
3375
3385
  scores: scores ?? {},
3376
- metrics
3386
+ metrics: metrics ?? {}
3377
3387
  };
3378
3388
  }
3379
3389
  /**