@arizeai/phoenix-client 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/README.md +118 -0
  2. package/dist/esm/client.d.ts +13 -1
  3. package/dist/esm/client.d.ts.map +1 -1
  4. package/dist/esm/client.js +4 -1
  5. package/dist/esm/client.js.map +1 -1
  6. package/dist/esm/datasets/appendDatasetExamples.d.ts +21 -0
  7. package/dist/esm/datasets/appendDatasetExamples.d.ts.map +1 -0
  8. package/dist/esm/datasets/appendDatasetExamples.js +32 -0
  9. package/dist/esm/datasets/appendDatasetExamples.js.map +1 -0
  10. package/dist/esm/datasets/createDataset.d.ts +25 -0
  11. package/dist/esm/datasets/createDataset.d.ts.map +1 -0
  12. package/dist/esm/datasets/createDataset.js +34 -0
  13. package/dist/esm/datasets/createDataset.js.map +1 -0
  14. package/dist/esm/datasets/getDataset.d.ts +10 -0
  15. package/dist/esm/datasets/getDataset.d.ts.map +1 -0
  16. package/dist/esm/datasets/getDataset.js +18 -0
  17. package/dist/esm/datasets/getDataset.js.map +1 -0
  18. package/dist/esm/datasets/getDatasetExamples.d.ts +10 -0
  19. package/dist/esm/datasets/getDatasetExamples.d.ts.map +1 -0
  20. package/dist/esm/datasets/getDatasetExamples.js +25 -0
  21. package/dist/esm/datasets/getDatasetExamples.js.map +1 -0
  22. package/dist/esm/datasets/getDatasetInfo.d.ts +11 -0
  23. package/dist/esm/datasets/getDatasetInfo.d.ts.map +1 -0
  24. package/dist/esm/datasets/getDatasetInfo.js +25 -0
  25. package/dist/esm/datasets/getDatasetInfo.js.map +1 -0
  26. package/dist/esm/datasets/index.d.ts +7 -0
  27. package/dist/esm/datasets/index.d.ts.map +1 -0
  28. package/dist/esm/datasets/index.js +7 -0
  29. package/dist/esm/datasets/index.js.map +1 -0
  30. package/dist/esm/datasets/listDatasets.d.ts +23 -0
  31. package/dist/esm/datasets/listDatasets.d.ts.map +1 -0
  32. package/dist/esm/datasets/listDatasets.js +26 -0
  33. package/dist/esm/datasets/listDatasets.js.map +1 -0
  34. package/dist/esm/experiments/getExperiment.d.ts +14 -0
  35. package/dist/esm/experiments/getExperiment.d.ts.map +1 -0
  36. package/dist/esm/experiments/getExperiment.js +25 -0
  37. package/dist/esm/experiments/getExperiment.js.map +1 -0
  38. package/dist/esm/experiments/getExperimentInfo.d.ts +13 -0
  39. package/dist/esm/experiments/getExperimentInfo.d.ts.map +1 -0
  40. package/dist/esm/experiments/getExperimentInfo.js +24 -0
  41. package/dist/esm/experiments/getExperimentInfo.js.map +1 -0
  42. package/dist/esm/experiments/getExperimentRuns.d.ts +15 -0
  43. package/dist/esm/experiments/getExperimentRuns.d.ts.map +1 -0
  44. package/dist/esm/experiments/getExperimentRuns.js +33 -0
  45. package/dist/esm/experiments/getExperimentRuns.js.map +1 -0
  46. package/dist/esm/experiments/index.d.ts +3 -0
  47. package/dist/esm/experiments/index.d.ts.map +1 -1
  48. package/dist/esm/experiments/index.js +3 -0
  49. package/dist/esm/experiments/index.js.map +1 -1
  50. package/dist/esm/experiments/instrumention.d.ts +18 -0
  51. package/dist/esm/experiments/instrumention.d.ts.map +1 -0
  52. package/dist/esm/experiments/instrumention.js +34 -0
  53. package/dist/esm/experiments/instrumention.js.map +1 -0
  54. package/dist/esm/experiments/runExperiment.d.ts +24 -21
  55. package/dist/esm/experiments/runExperiment.d.ts.map +1 -1
  56. package/dist/esm/experiments/runExperiment.js +221 -108
  57. package/dist/esm/experiments/runExperiment.js.map +1 -1
  58. package/dist/esm/schemas/llm/anthropic/converters.d.ts +28 -28
  59. package/dist/esm/schemas/llm/anthropic/messagePartSchemas.d.ts +8 -8
  60. package/dist/esm/schemas/llm/anthropic/messageSchemas.d.ts +24 -24
  61. package/dist/esm/schemas/llm/anthropic/toolCallSchemas.d.ts +8 -8
  62. package/dist/esm/schemas/llm/constants.d.ts +6 -6
  63. package/dist/esm/schemas/llm/converters.d.ts +24 -24
  64. package/dist/esm/schemas/llm/openai/converters.d.ts +6 -6
  65. package/dist/esm/schemas/llm/schemas.d.ts +22 -22
  66. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  67. package/dist/esm/types/datasets.d.ts +33 -8
  68. package/dist/esm/types/datasets.d.ts.map +1 -1
  69. package/dist/esm/types/experiments.d.ts +17 -4
  70. package/dist/esm/types/experiments.d.ts.map +1 -1
  71. package/dist/esm/utils/ensureString.d.ts +8 -0
  72. package/dist/esm/utils/ensureString.d.ts.map +1 -0
  73. package/dist/esm/utils/ensureString.js +14 -0
  74. package/dist/esm/utils/ensureString.js.map +1 -0
  75. package/dist/esm/utils/objectAsAttributes.d.ts +3 -0
  76. package/dist/esm/utils/objectAsAttributes.d.ts.map +1 -0
  77. package/dist/esm/utils/objectAsAttributes.js +4 -0
  78. package/dist/esm/utils/objectAsAttributes.js.map +1 -0
  79. package/dist/src/client.d.ts +13 -1
  80. package/dist/src/client.d.ts.map +1 -1
  81. package/dist/src/client.js +1 -1
  82. package/dist/src/client.js.map +1 -1
  83. package/dist/src/datasets/appendDatasetExamples.d.ts +21 -0
  84. package/dist/src/datasets/appendDatasetExamples.d.ts.map +1 -0
  85. package/dist/src/datasets/appendDatasetExamples.js +50 -0
  86. package/dist/src/datasets/appendDatasetExamples.js.map +1 -0
  87. package/dist/src/datasets/createDataset.d.ts +25 -0
  88. package/dist/src/datasets/createDataset.d.ts.map +1 -0
  89. package/dist/src/datasets/createDataset.js +52 -0
  90. package/dist/src/datasets/createDataset.js.map +1 -0
  91. package/dist/src/datasets/getDataset.d.ts +10 -0
  92. package/dist/src/datasets/getDataset.d.ts.map +1 -0
  93. package/dist/src/datasets/getDataset.js +29 -0
  94. package/dist/src/datasets/getDataset.js.map +1 -0
  95. package/dist/src/datasets/getDatasetExamples.d.ts +10 -0
  96. package/dist/src/datasets/getDatasetExamples.d.ts.map +1 -0
  97. package/dist/src/datasets/getDatasetExamples.js +40 -0
  98. package/dist/src/datasets/getDatasetExamples.js.map +1 -0
  99. package/dist/src/datasets/getDatasetInfo.d.ts +11 -0
  100. package/dist/src/datasets/getDatasetInfo.d.ts.map +1 -0
  101. package/dist/src/datasets/getDatasetInfo.js +43 -0
  102. package/dist/src/datasets/getDatasetInfo.js.map +1 -0
  103. package/dist/src/datasets/index.d.ts +7 -0
  104. package/dist/src/datasets/index.d.ts.map +1 -0
  105. package/dist/src/datasets/index.js +23 -0
  106. package/dist/src/datasets/index.js.map +1 -0
  107. package/dist/src/datasets/listDatasets.d.ts +23 -0
  108. package/dist/src/datasets/listDatasets.d.ts.map +1 -0
  109. package/dist/src/datasets/listDatasets.js +40 -0
  110. package/dist/src/datasets/listDatasets.js.map +1 -0
  111. package/dist/src/experiments/getExperiment.d.ts +14 -0
  112. package/dist/src/experiments/getExperiment.d.ts.map +1 -0
  113. package/dist/src/experiments/getExperiment.js +36 -0
  114. package/dist/src/experiments/getExperiment.js.map +1 -0
  115. package/dist/src/experiments/getExperimentInfo.d.ts +13 -0
  116. package/dist/src/experiments/getExperimentInfo.d.ts.map +1 -0
  117. package/dist/src/experiments/getExperimentInfo.js +41 -0
  118. package/dist/src/experiments/getExperimentInfo.js.map +1 -0
  119. package/dist/src/experiments/getExperimentRuns.d.ts +15 -0
  120. package/dist/src/experiments/getExperimentRuns.d.ts.map +1 -0
  121. package/dist/src/experiments/getExperimentRuns.js +50 -0
  122. package/dist/src/experiments/getExperimentRuns.js.map +1 -0
  123. package/dist/src/experiments/index.d.ts +3 -0
  124. package/dist/src/experiments/index.d.ts.map +1 -1
  125. package/dist/src/experiments/index.js +3 -0
  126. package/dist/src/experiments/index.js.map +1 -1
  127. package/dist/src/experiments/instrumention.d.ts +18 -0
  128. package/dist/src/experiments/instrumention.d.ts.map +1 -0
  129. package/dist/src/experiments/instrumention.js +38 -0
  130. package/dist/src/experiments/instrumention.js.map +1 -0
  131. package/dist/src/experiments/runExperiment.d.ts +24 -21
  132. package/dist/src/experiments/runExperiment.d.ts.map +1 -1
  133. package/dist/src/experiments/runExperiment.js +222 -111
  134. package/dist/src/experiments/runExperiment.js.map +1 -1
  135. package/dist/src/schemas/llm/anthropic/converters.d.ts +28 -28
  136. package/dist/src/schemas/llm/anthropic/messagePartSchemas.d.ts +8 -8
  137. package/dist/src/schemas/llm/anthropic/messageSchemas.d.ts +24 -24
  138. package/dist/src/schemas/llm/anthropic/toolCallSchemas.d.ts +8 -8
  139. package/dist/src/schemas/llm/constants.d.ts +6 -6
  140. package/dist/src/schemas/llm/converters.d.ts +24 -24
  141. package/dist/src/schemas/llm/openai/converters.d.ts +6 -6
  142. package/dist/src/schemas/llm/schemas.d.ts +22 -22
  143. package/dist/src/types/datasets.d.ts +33 -8
  144. package/dist/src/types/datasets.d.ts.map +1 -1
  145. package/dist/src/types/experiments.d.ts +17 -4
  146. package/dist/src/types/experiments.d.ts.map +1 -1
  147. package/dist/src/utils/ensureString.d.ts +8 -0
  148. package/dist/src/utils/ensureString.d.ts.map +1 -0
  149. package/dist/src/utils/ensureString.js +18 -0
  150. package/dist/src/utils/ensureString.js.map +1 -0
  151. package/dist/src/utils/objectAsAttributes.d.ts +3 -0
  152. package/dist/src/utils/objectAsAttributes.d.ts.map +1 -0
  153. package/dist/src/utils/objectAsAttributes.js +7 -0
  154. package/dist/src/utils/objectAsAttributes.js.map +1 -0
  155. package/dist/tsconfig.tsbuildinfo +1 -1
  156. package/package.json +13 -1
  157. package/src/client.ts +4 -1
  158. package/src/datasets/appendDatasetExamples.ts +55 -0
  159. package/src/datasets/createDataset.ts +60 -0
  160. package/src/datasets/getDataset.ts +27 -0
  161. package/src/datasets/getDatasetExamples.ts +34 -0
  162. package/src/datasets/getDatasetInfo.ts +34 -0
  163. package/src/datasets/index.ts +6 -0
  164. package/src/datasets/listDatasets.ts +37 -0
  165. package/src/experiments/getExperiment.ts +40 -0
  166. package/src/experiments/getExperimentInfo.ts +39 -0
  167. package/src/experiments/getExperimentRuns.ts +45 -0
  168. package/src/experiments/index.ts +3 -0
  169. package/src/experiments/instrumention.ts +52 -0
  170. package/src/experiments/runExperiment.ts +277 -133
  171. package/src/types/datasets.ts +35 -9
  172. package/src/types/experiments.ts +19 -4
  173. package/src/utils/ensureString.ts +14 -0
  174. package/src/utils/objectAsAttributes.ts +9 -0
  175. package/dist/esm/utils/getDatasetBySelector.d.ts +0 -25
  176. package/dist/esm/utils/getDatasetBySelector.d.ts.map +0 -1
  177. package/dist/esm/utils/getDatasetBySelector.js +0 -37
  178. package/dist/esm/utils/getDatasetBySelector.js.map +0 -1
  179. package/dist/src/utils/getDatasetBySelector.d.ts +0 -25
  180. package/dist/src/utils/getDatasetBySelector.d.ts.map +0 -1
  181. package/dist/src/utils/getDatasetBySelector.js +0 -47
  182. package/dist/src/utils/getDatasetBySelector.js.map +0 -1
  183. package/src/utils/getDatasetBySelector.ts +0 -55
@@ -0,0 +1,15 @@
1
+ import { ClientFn } from "../types/core.js";
2
+ import { ExperimentRun } from "../types/experiments.js";
3
+ export type GetExperimentRunsParams = ClientFn & {
4
+ /**
5
+ * The experiment ID.
6
+ */
7
+ experimentId: string;
8
+ };
9
+ /**
10
+ * A function that gets the runs (e.g. the results) of a experiment
11
+ */
12
+ export declare function getExperimentRuns({ client: _client, experimentId, }: GetExperimentRunsParams): Promise<{
13
+ runs: ExperimentRun[];
14
+ }>;
15
+ //# sourceMappingURL=getExperimentRuns.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getExperimentRuns.d.ts","sourceRoot":"","sources":["../../../src/experiments/getExperimentRuns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,MAAM,MAAM,uBAAuB,GAAG,QAAQ,GAAG;IAC/C;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,MAAM,EAAE,OAAO,EACf,YAAY,GACb,EAAE,uBAAuB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,aAAa,EAAE,CAAA;CAAE,CAAC,CA0B9D"}
@@ -0,0 +1,33 @@
1
+ import { createClient } from "../client.js";
2
+ import invariant from "tiny-invariant";
3
+ /**
4
+ * A function that gets the runs (e.g. the results) of a experiment
5
+ */
6
+ export async function getExperimentRuns({ client: _client, experimentId, }) {
7
+ const client = _client || createClient();
8
+ const getRunsPromise = client.GET("/v1/experiments/{experiment_id}/runs", {
9
+ params: {
10
+ path: {
11
+ experiment_id: experimentId,
12
+ },
13
+ },
14
+ });
15
+ const [experimentRunResponse] = await Promise.all([getRunsPromise]);
16
+ const { data: { data: experimentRunsData } = {} } = experimentRunResponse;
17
+ invariant(experimentRunsData, "Failed to retrieve experiment runs");
18
+ return {
19
+ runs: experimentRunsData.map((run) => {
20
+ return {
21
+ id: run.id,
22
+ traceId: run.trace_id || null,
23
+ experimentId: run.experiment_id,
24
+ datasetExampleId: run.dataset_example_id,
25
+ startTime: new Date(run.start_time),
26
+ endTime: new Date(run.end_time),
27
+ output: run.output,
28
+ error: run.error || null,
29
+ };
30
+ }),
31
+ };
32
+ }
33
+ //# sourceMappingURL=getExperimentRuns.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getExperimentRuns.js","sourceRoot":"","sources":["../../../src/experiments/getExperimentRuns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,SAAS,MAAM,gBAAgB,CAAC;AAWvC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,MAAM,EAAE,OAAO,EACf,YAAY,GACY;IACxB,MAAM,MAAM,GAAG,OAAO,IAAI,YAAY,EAAE,CAAC;IACzC,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAC,sCAAsC,EAAE;QACxE,MAAM,EAAE;YACN,IAAI,EAAE;gBACJ,aAAa,EAAE,YAAY;aAC5B;SACF;KACF,CAAC,CAAC;IACH,MAAM,CAAC,qBAAqB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;IACpE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,EAAE,GAAG,qBAAqB,CAAC;IAC1E,SAAS,CAAC,kBAAkB,EAAE,oCAAoC,CAAC,CAAC;IACpE,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACnC,OAAO;gBACL,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,OAAO,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;gBAC7B,YAAY,EAAE,GAAG,CAAC,aAAa;gBAC/B,gBAAgB,EAAE,GAAG,CAAC,kBAAkB;gBACxC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;gBACnC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC/B,MAAM,EAAE,GAAG,CAAC,MAAiC;gBAC7C,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI;aACzB,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC"}
@@ -1,2 +1,5 @@
1
+ export * from "./getExperimentInfo.js";
2
+ export * from "./getExperiment.js";
3
+ export * from "./getExperimentRuns.js";
1
4
  export * from "./runExperiment.js";
2
5
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/experiments/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/experiments/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC"}
@@ -1,2 +1,5 @@
1
+ export * from "./getExperimentInfo.js";
2
+ export * from "./getExperiment.js";
3
+ export * from "./getExperimentRuns.js";
1
4
  export * from "./runExperiment.js";
2
5
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/experiments/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/experiments/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC"}
@@ -0,0 +1,18 @@
1
+ import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
2
+ import { HeadersOptions } from "openapi-fetch";
3
+ /**
4
+ * Creates a provider that exports traces to Phoenix.
5
+ */
6
+ export declare function createProvider({ projectName, baseUrl, headers, }: {
7
+ projectName: string;
8
+ headers: HeadersOptions;
9
+ /**
10
+ * The base URL of the Phoenix. Doesn't include the /v1/traces path.
11
+ */
12
+ baseUrl: string;
13
+ }): NodeTracerProvider;
14
+ /**
15
+ * For dry runs we create a provider that doesn't export traces.
16
+ */
17
+ export declare function createNoOpProvider(): NodeTracerProvider;
18
+ //# sourceMappingURL=instrumention.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"instrumention.d.ts","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,GACR,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB,sBAoBA;AAED;;GAEG;AACH,wBAAgB,kBAAkB,uBAIjC"}
@@ -0,0 +1,34 @@
1
+ import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
2
+ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
3
+ import { resourceFromAttributes } from "@opentelemetry/resources";
4
+ import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
5
+ import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
6
+ import { SEMRESATTRS_PROJECT_NAME } from "@arizeai/openinference-semantic-conventions";
7
+ /**
8
+ * Creates a provider that exports traces to Phoenix.
9
+ */
10
+ export function createProvider({ projectName, baseUrl, headers, }) {
11
+ diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR);
12
+ const provider = new NodeTracerProvider({
13
+ resource: resourceFromAttributes({
14
+ [SEMRESATTRS_PROJECT_NAME]: projectName,
15
+ }),
16
+ spanProcessors: [
17
+ new SimpleSpanProcessor(new OTLPTraceExporter({
18
+ url: `${baseUrl}/v1/traces`,
19
+ headers: Array.isArray(headers)
20
+ ? Object.fromEntries(headers)
21
+ : headers,
22
+ })),
23
+ ],
24
+ });
25
+ return provider;
26
+ }
27
+ /**
28
+ * For dry runs we create a provider that doesn't export traces.
29
+ */
30
+ export function createNoOpProvider() {
31
+ const provider = new NodeTracerProvider({});
32
+ return provider;
33
+ }
34
+ //# sourceMappingURL=instrumention.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"instrumention.js","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,MAAM,6CAA6C,CAAC;AAGvF;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,GAQR;IACC,IAAI,CAAC,SAAS,CAAC,IAAI,iBAAiB,EAAE,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;QACtC,QAAQ,EAAE,sBAAsB,CAAC;YAC/B,CAAC,wBAAwB,CAAC,EAAE,WAAW;SACxC,CAAC;QACF,cAAc,EAAE;YACd,IAAI,mBAAmB,CACrB,IAAI,iBAAiB,CAAC;gBACpB,GAAG,EAAE,GAAG,OAAO,YAAY;gBAC3B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;oBAC7B,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC;oBAC7B,CAAC,CAAC,OAAO;aACZ,CAAC,CACH;SACF;KACF,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAE5C,OAAO,QAAQ,CAAC;AAClB,CAAC"}
@@ -1,6 +1,6 @@
1
1
  import { type PhoenixClient } from "../client.js";
2
2
  import { ClientFn } from "../types/core.js";
3
- import { Dataset, Example } from "../types/datasets.js";
3
+ import { DatasetSelector } from "../types/datasets.js";
4
4
  import type { Evaluator, ExperimentTask, RanExperiment } from "../types/experiments.js";
5
5
  import { type Logger } from "../types/logger.js";
6
6
  import { AnnotatorKind } from "../types/annotations.js";
@@ -21,12 +21,13 @@ export type RunExperimentParams = ClientFn & {
21
21
  experimentDescription?: string;
22
22
  /**
23
23
  * Experiment metadata
24
+ * E.x. modelName
24
25
  */
25
26
  experimentMetadata?: Record<string, unknown>;
26
27
  /**
27
28
  * The dataset to run the experiment on
28
29
  */
29
- dataset: Dataset | string | Example[];
30
+ dataset: DatasetSelector;
30
31
  /**
31
32
  * The task to run
32
33
  */
@@ -35,10 +36,6 @@ export type RunExperimentParams = ClientFn & {
35
36
  * The evaluators to use
36
37
  */
37
38
  evaluators?: Evaluator[];
38
- /**
39
- * The project under which the experiment task traces are recorded
40
- */
41
- projectName?: string;
42
39
  /**
43
40
  * The logger to use
44
41
  */
@@ -58,7 +55,23 @@ export type RunExperimentParams = ClientFn & {
58
55
  dryRun?: number | boolean;
59
56
  };
60
57
  /**
61
- * Run an experiment.
58
+ * Runs an experiment using a given set of dataset of examples.
59
+ *
60
+ * An experiment is a user-defined task that runs on each example in a dataset. The results from
61
+ * each experiment can be evaluated using any number of evaluators to measure the behavior of the
62
+ * task. The experiment and evaluation results are stored in the Phoenix database for comparison
63
+ * and analysis.
64
+ *
65
+ * A `task` is either a sync or async function that returns a JSON serializable
66
+ * output. If the `task` is a function of one argument then that argument will be bound to the
67
+ * `input` field of the dataset example. Alternatively, the `task` can be a function of any
68
+ * combination of specific argument names that will be bound to special values:
69
+ *
70
+ * - `input`: The input field of the dataset example
71
+ * - `expected`: The expected or reference output of the dataset example
72
+ * - `reference`: An alias for `expected`
73
+ * - `metadata`: Metadata associated with the dataset example
74
+ * - `example`: The dataset `Example` object with all associated fields
62
75
  *
63
76
  * @example
64
77
  * ```ts
@@ -68,14 +81,12 @@ export type RunExperimentParams = ClientFn & {
68
81
  * dataset: "my-dataset",
69
82
  * task: async (example) => example.input,
70
83
  * evaluators: [
71
- * asEvaluator("my-evaluator", "CODE", async (params) => params.output),
84
+ * asEvaluator({ name: "my-evaluator", kind: "CODE", evaluate: async (params) => params.output }),
72
85
  * ],
73
86
  * });
74
87
  * ```
75
- *
76
- * @experimental This feature is not complete, and will change in the future.
77
88
  */
78
- export declare function runExperiment({ experimentName: _experimentName, experimentDescription, experimentMetadata, client: _client, dataset: _dataset, task, evaluators, projectName, logger, record, concurrency, dryRun, }: RunExperimentParams): Promise<RanExperiment>;
89
+ export declare function runExperiment({ experimentName, experimentDescription, experimentMetadata, client: _client, dataset: DatasetSelector, task, evaluators, logger, record, concurrency, dryRun, }: RunExperimentParams): Promise<RanExperiment>;
79
90
  /**
80
91
  * Evaluate an experiment.
81
92
  *
@@ -84,7 +95,6 @@ export declare function runExperiment({ experimentName: _experimentName, experim
84
95
  export declare function evaluateExperiment({ experiment, evaluators, client: _client, logger, concurrency, dryRun, }: {
85
96
  /**
86
97
  * The experiment to evaluate
87
- * @todo also accept Experiment, and attempt to fetch the runs from the server
88
98
  **/
89
99
  experiment: RanExperiment;
90
100
  /** The evaluators to use */
@@ -92,9 +102,9 @@ export declare function evaluateExperiment({ experiment, evaluators, client: _cl
92
102
  /** The client to use */
93
103
  client?: PhoenixClient;
94
104
  /** The logger to use */
95
- logger: Logger;
105
+ logger?: Logger;
96
106
  /** The number of evaluators to run in parallel */
97
- concurrency: number;
107
+ concurrency?: number;
98
108
  /**
99
109
  * Whether to run the evaluation as a dry run
100
110
  * If a number is provided, the evaluation will be run for the first n runs
@@ -116,11 +126,4 @@ export declare function asEvaluator({ name, kind, evaluate, }: {
116
126
  kind: AnnotatorKind;
117
127
  evaluate: Evaluator["evaluate"];
118
128
  }): Evaluator;
119
- /**
120
- * Generate a unique id.
121
- *
122
- * @deprecated Use id generated by phoenix instead.
123
- * @returns A unique id.
124
- */
125
- export declare function id(): string;
126
129
  //# sourceMappingURL=runExperiment.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"runExperiment.d.ts","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EACV,SAAS,EAKT,cAAc,EACd,aAAa,EACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAI9C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,QAAQ,GAAG;IAC3C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7C;;OAEG;IACH,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,EAAE,CAAC;IACtC;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;OAEG;IACH,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,aAAa,CAAC,EAClC,cAAc,EAAE,eAAe,EAC/B,qBAAqB,EACrB,kBAAkB,EAClB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,QAAQ,EACjB,IAAI,EACJ,UAAU,EACV,WAAuB,EACvB,MAAgB,EAChB,MAAa,EACb,WAAe,EACf,MAAc,GACf,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAmG9C;AA4FD;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAM,EACN,WAAe,EACf,MAAc,GACf,EAAE;IACD;;;QAGI;IACJ,UAAU,EAAE,aAAa,CAAC;IAC1B,4BAA4B;IAC5B,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,wBAAwB;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB;;;;SAIK;IACL,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CAC3B,GAAG,OAAO,CAAC,aAAa,CAAC,CA6FzB;AAmDD;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,EAC1B,IAAI,EACJ,IAAI,EACJ,QAAQ,GACT,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,aAAa,CAAC;IACpB,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;CACjC,GAAG,SAAS,CAMZ;AAID;;;;;GAKG;AACH,wBAAgB,EAAE,IAAI,MAAM,CAK3B"}
1
+ {"version":3,"file":"runExperiment.d.ts","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAEL,eAAe,EAGhB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,SAAS,EAKT,cAAc,EACd,aAAa,EACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAI9C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAYrD;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,QAAQ,GAAG;IAC3C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7C;;OAEG;IACH,OAAO,EAAE,eAAe,CAAC;IACzB;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;OAEG;IACH,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAsB,aAAa,CAAC,EAClC,cAAc,EACd,qBAAqB,EACrB,kBAAuB,EACvB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,eAAe,EACxB,IAAI,EACJ,UAAU,EACV,MAAgB,EAChB,MAAa,EACb,WAAe,EACf,MAAc,GACf,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAoH9C;AAwHD;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAgB,EAChB,WAAe,EACf,MAAc,GACf,EAAE;IACD;;QAEI;IACJ,UAAU,EAAE,aAAa,CAAC;IAC1B,4BAA4B;IAC5B,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,wBAAwB;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;SAIK;IACL,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CAC3B,GAAG,OAAO,CAAC,aAAa,CAAC,CAgKzB;AA8DD;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,EAC1B,IAAI,EACJ,IAAI,EACJ,QAAQ,GACT,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,aAAa,CAAC;IACpB,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;CACjC,GAAG,SAAS,CAMZ"}