@arizeai/phoenix-client 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -0
- package/dist/esm/client.d.ts +13 -1
- package/dist/esm/client.d.ts.map +1 -1
- package/dist/esm/client.js +4 -1
- package/dist/esm/client.js.map +1 -1
- package/dist/esm/datasets/appendDatasetExamples.d.ts +21 -0
- package/dist/esm/datasets/appendDatasetExamples.d.ts.map +1 -0
- package/dist/esm/datasets/appendDatasetExamples.js +32 -0
- package/dist/esm/datasets/appendDatasetExamples.js.map +1 -0
- package/dist/esm/datasets/createDataset.d.ts +25 -0
- package/dist/esm/datasets/createDataset.d.ts.map +1 -0
- package/dist/esm/datasets/createDataset.js +34 -0
- package/dist/esm/datasets/createDataset.js.map +1 -0
- package/dist/esm/datasets/getDataset.d.ts +10 -0
- package/dist/esm/datasets/getDataset.d.ts.map +1 -0
- package/dist/esm/datasets/getDataset.js +18 -0
- package/dist/esm/datasets/getDataset.js.map +1 -0
- package/dist/esm/datasets/getDatasetExamples.d.ts +10 -0
- package/dist/esm/datasets/getDatasetExamples.d.ts.map +1 -0
- package/dist/esm/datasets/getDatasetExamples.js +25 -0
- package/dist/esm/datasets/getDatasetExamples.js.map +1 -0
- package/dist/esm/datasets/getDatasetInfo.d.ts +11 -0
- package/dist/esm/datasets/getDatasetInfo.d.ts.map +1 -0
- package/dist/esm/datasets/getDatasetInfo.js +25 -0
- package/dist/esm/datasets/getDatasetInfo.js.map +1 -0
- package/dist/esm/datasets/index.d.ts +7 -0
- package/dist/esm/datasets/index.d.ts.map +1 -0
- package/dist/esm/datasets/index.js +7 -0
- package/dist/esm/datasets/index.js.map +1 -0
- package/dist/esm/datasets/listDatasets.d.ts +23 -0
- package/dist/esm/datasets/listDatasets.d.ts.map +1 -0
- package/dist/esm/datasets/listDatasets.js +26 -0
- package/dist/esm/datasets/listDatasets.js.map +1 -0
- package/dist/esm/experiments/getExperiment.d.ts +14 -0
- package/dist/esm/experiments/getExperiment.d.ts.map +1 -0
- package/dist/esm/experiments/getExperiment.js +25 -0
- package/dist/esm/experiments/getExperiment.js.map +1 -0
- package/dist/esm/experiments/getExperimentInfo.d.ts +13 -0
- package/dist/esm/experiments/getExperimentInfo.d.ts.map +1 -0
- package/dist/esm/experiments/getExperimentInfo.js +24 -0
- package/dist/esm/experiments/getExperimentInfo.js.map +1 -0
- package/dist/esm/experiments/getExperimentRuns.d.ts +15 -0
- package/dist/esm/experiments/getExperimentRuns.d.ts.map +1 -0
- package/dist/esm/experiments/getExperimentRuns.js +33 -0
- package/dist/esm/experiments/getExperimentRuns.js.map +1 -0
- package/dist/esm/experiments/index.d.ts +3 -0
- package/dist/esm/experiments/index.d.ts.map +1 -1
- package/dist/esm/experiments/index.js +3 -0
- package/dist/esm/experiments/index.js.map +1 -1
- package/dist/esm/experiments/instrumention.d.ts +18 -0
- package/dist/esm/experiments/instrumention.d.ts.map +1 -0
- package/dist/esm/experiments/instrumention.js +34 -0
- package/dist/esm/experiments/instrumention.js.map +1 -0
- package/dist/esm/experiments/runExperiment.d.ts +24 -21
- package/dist/esm/experiments/runExperiment.d.ts.map +1 -1
- package/dist/esm/experiments/runExperiment.js +221 -108
- package/dist/esm/experiments/runExperiment.js.map +1 -1
- package/dist/esm/schemas/llm/anthropic/converters.d.ts +28 -28
- package/dist/esm/schemas/llm/anthropic/messagePartSchemas.d.ts +8 -8
- package/dist/esm/schemas/llm/anthropic/messageSchemas.d.ts +24 -24
- package/dist/esm/schemas/llm/anthropic/toolCallSchemas.d.ts +8 -8
- package/dist/esm/schemas/llm/constants.d.ts +6 -6
- package/dist/esm/schemas/llm/converters.d.ts +24 -24
- package/dist/esm/schemas/llm/openai/converters.d.ts +6 -6
- package/dist/esm/schemas/llm/schemas.d.ts +22 -22
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/esm/types/datasets.d.ts +33 -8
- package/dist/esm/types/datasets.d.ts.map +1 -1
- package/dist/esm/types/experiments.d.ts +17 -4
- package/dist/esm/types/experiments.d.ts.map +1 -1
- package/dist/esm/utils/ensureString.d.ts +8 -0
- package/dist/esm/utils/ensureString.d.ts.map +1 -0
- package/dist/esm/utils/ensureString.js +14 -0
- package/dist/esm/utils/ensureString.js.map +1 -0
- package/dist/esm/utils/objectAsAttributes.d.ts +3 -0
- package/dist/esm/utils/objectAsAttributes.d.ts.map +1 -0
- package/dist/esm/utils/objectAsAttributes.js +4 -0
- package/dist/esm/utils/objectAsAttributes.js.map +1 -0
- package/dist/src/client.d.ts +13 -1
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +1 -1
- package/dist/src/client.js.map +1 -1
- package/dist/src/datasets/appendDatasetExamples.d.ts +21 -0
- package/dist/src/datasets/appendDatasetExamples.d.ts.map +1 -0
- package/dist/src/datasets/appendDatasetExamples.js +50 -0
- package/dist/src/datasets/appendDatasetExamples.js.map +1 -0
- package/dist/src/datasets/createDataset.d.ts +25 -0
- package/dist/src/datasets/createDataset.d.ts.map +1 -0
- package/dist/src/datasets/createDataset.js +52 -0
- package/dist/src/datasets/createDataset.js.map +1 -0
- package/dist/src/datasets/getDataset.d.ts +10 -0
- package/dist/src/datasets/getDataset.d.ts.map +1 -0
- package/dist/src/datasets/getDataset.js +29 -0
- package/dist/src/datasets/getDataset.js.map +1 -0
- package/dist/src/datasets/getDatasetExamples.d.ts +10 -0
- package/dist/src/datasets/getDatasetExamples.d.ts.map +1 -0
- package/dist/src/datasets/getDatasetExamples.js +40 -0
- package/dist/src/datasets/getDatasetExamples.js.map +1 -0
- package/dist/src/datasets/getDatasetInfo.d.ts +11 -0
- package/dist/src/datasets/getDatasetInfo.d.ts.map +1 -0
- package/dist/src/datasets/getDatasetInfo.js +43 -0
- package/dist/src/datasets/getDatasetInfo.js.map +1 -0
- package/dist/src/datasets/index.d.ts +7 -0
- package/dist/src/datasets/index.d.ts.map +1 -0
- package/dist/src/datasets/index.js +23 -0
- package/dist/src/datasets/index.js.map +1 -0
- package/dist/src/datasets/listDatasets.d.ts +23 -0
- package/dist/src/datasets/listDatasets.d.ts.map +1 -0
- package/dist/src/datasets/listDatasets.js +40 -0
- package/dist/src/datasets/listDatasets.js.map +1 -0
- package/dist/src/experiments/getExperiment.d.ts +14 -0
- package/dist/src/experiments/getExperiment.d.ts.map +1 -0
- package/dist/src/experiments/getExperiment.js +36 -0
- package/dist/src/experiments/getExperiment.js.map +1 -0
- package/dist/src/experiments/getExperimentInfo.d.ts +13 -0
- package/dist/src/experiments/getExperimentInfo.d.ts.map +1 -0
- package/dist/src/experiments/getExperimentInfo.js +41 -0
- package/dist/src/experiments/getExperimentInfo.js.map +1 -0
- package/dist/src/experiments/getExperimentRuns.d.ts +15 -0
- package/dist/src/experiments/getExperimentRuns.d.ts.map +1 -0
- package/dist/src/experiments/getExperimentRuns.js +50 -0
- package/dist/src/experiments/getExperimentRuns.js.map +1 -0
- package/dist/src/experiments/index.d.ts +3 -0
- package/dist/src/experiments/index.d.ts.map +1 -1
- package/dist/src/experiments/index.js +3 -0
- package/dist/src/experiments/index.js.map +1 -1
- package/dist/src/experiments/instrumention.d.ts +18 -0
- package/dist/src/experiments/instrumention.d.ts.map +1 -0
- package/dist/src/experiments/instrumention.js +38 -0
- package/dist/src/experiments/instrumention.js.map +1 -0
- package/dist/src/experiments/runExperiment.d.ts +24 -21
- package/dist/src/experiments/runExperiment.d.ts.map +1 -1
- package/dist/src/experiments/runExperiment.js +222 -111
- package/dist/src/experiments/runExperiment.js.map +1 -1
- package/dist/src/schemas/llm/anthropic/converters.d.ts +28 -28
- package/dist/src/schemas/llm/anthropic/messagePartSchemas.d.ts +8 -8
- package/dist/src/schemas/llm/anthropic/messageSchemas.d.ts +24 -24
- package/dist/src/schemas/llm/anthropic/toolCallSchemas.d.ts +8 -8
- package/dist/src/schemas/llm/constants.d.ts +6 -6
- package/dist/src/schemas/llm/converters.d.ts +24 -24
- package/dist/src/schemas/llm/openai/converters.d.ts +6 -6
- package/dist/src/schemas/llm/schemas.d.ts +22 -22
- package/dist/src/types/datasets.d.ts +33 -8
- package/dist/src/types/datasets.d.ts.map +1 -1
- package/dist/src/types/experiments.d.ts +17 -4
- package/dist/src/types/experiments.d.ts.map +1 -1
- package/dist/src/utils/ensureString.d.ts +8 -0
- package/dist/src/utils/ensureString.d.ts.map +1 -0
- package/dist/src/utils/ensureString.js +18 -0
- package/dist/src/utils/ensureString.js.map +1 -0
- package/dist/src/utils/objectAsAttributes.d.ts +3 -0
- package/dist/src/utils/objectAsAttributes.d.ts.map +1 -0
- package/dist/src/utils/objectAsAttributes.js +7 -0
- package/dist/src/utils/objectAsAttributes.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +13 -1
- package/src/client.ts +4 -1
- package/src/datasets/appendDatasetExamples.ts +55 -0
- package/src/datasets/createDataset.ts +60 -0
- package/src/datasets/getDataset.ts +27 -0
- package/src/datasets/getDatasetExamples.ts +34 -0
- package/src/datasets/getDatasetInfo.ts +34 -0
- package/src/datasets/index.ts +6 -0
- package/src/datasets/listDatasets.ts +37 -0
- package/src/experiments/getExperiment.ts +40 -0
- package/src/experiments/getExperimentInfo.ts +39 -0
- package/src/experiments/getExperimentRuns.ts +45 -0
- package/src/experiments/index.ts +3 -0
- package/src/experiments/instrumention.ts +52 -0
- package/src/experiments/runExperiment.ts +277 -133
- package/src/types/datasets.ts +35 -9
- package/src/types/experiments.ts +19 -4
- package/src/utils/ensureString.ts +14 -0
- package/src/utils/objectAsAttributes.ts +9 -0
- package/dist/esm/utils/getDatasetBySelector.d.ts +0 -25
- package/dist/esm/utils/getDatasetBySelector.d.ts.map +0 -1
- package/dist/esm/utils/getDatasetBySelector.js +0 -37
- package/dist/esm/utils/getDatasetBySelector.js.map +0 -1
- package/dist/src/utils/getDatasetBySelector.d.ts +0 -25
- package/dist/src/utils/getDatasetBySelector.d.ts.map +0 -1
- package/dist/src/utils/getDatasetBySelector.js +0 -47
- package/dist/src/utils/getDatasetBySelector.js.map +0 -1
- package/src/utils/getDatasetBySelector.ts +0 -55
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { ClientFn } from "../types/core.js";
|
|
2
|
+
import { ExperimentRun } from "../types/experiments.js";
|
|
3
|
+
export type GetExperimentRunsParams = ClientFn & {
|
|
4
|
+
/**
|
|
5
|
+
* The experiment ID.
|
|
6
|
+
*/
|
|
7
|
+
experimentId: string;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* A function that gets the runs (e.g. the results) of a experiment
|
|
11
|
+
*/
|
|
12
|
+
export declare function getExperimentRuns({ client: _client, experimentId, }: GetExperimentRunsParams): Promise<{
|
|
13
|
+
runs: ExperimentRun[];
|
|
14
|
+
}>;
|
|
15
|
+
//# sourceMappingURL=getExperimentRuns.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getExperimentRuns.d.ts","sourceRoot":"","sources":["../../../src/experiments/getExperimentRuns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,MAAM,MAAM,uBAAuB,GAAG,QAAQ,GAAG;IAC/C;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,MAAM,EAAE,OAAO,EACf,YAAY,GACb,EAAE,uBAAuB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,aAAa,EAAE,CAAA;CAAE,CAAC,CA0B9D"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { createClient } from "../client.js";
|
|
2
|
+
import invariant from "tiny-invariant";
|
|
3
|
+
/**
|
|
4
|
+
* A function that gets the runs (e.g. the results) of a experiment
|
|
5
|
+
*/
|
|
6
|
+
export async function getExperimentRuns({ client: _client, experimentId, }) {
|
|
7
|
+
const client = _client || createClient();
|
|
8
|
+
const getRunsPromise = client.GET("/v1/experiments/{experiment_id}/runs", {
|
|
9
|
+
params: {
|
|
10
|
+
path: {
|
|
11
|
+
experiment_id: experimentId,
|
|
12
|
+
},
|
|
13
|
+
},
|
|
14
|
+
});
|
|
15
|
+
const [experimentRunResponse] = await Promise.all([getRunsPromise]);
|
|
16
|
+
const { data: { data: experimentRunsData } = {} } = experimentRunResponse;
|
|
17
|
+
invariant(experimentRunsData, "Failed to retrieve experiment runs");
|
|
18
|
+
return {
|
|
19
|
+
runs: experimentRunsData.map((run) => {
|
|
20
|
+
return {
|
|
21
|
+
id: run.id,
|
|
22
|
+
traceId: run.trace_id || null,
|
|
23
|
+
experimentId: run.experiment_id,
|
|
24
|
+
datasetExampleId: run.dataset_example_id,
|
|
25
|
+
startTime: new Date(run.start_time),
|
|
26
|
+
endTime: new Date(run.end_time),
|
|
27
|
+
output: run.output,
|
|
28
|
+
error: run.error || null,
|
|
29
|
+
};
|
|
30
|
+
}),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=getExperimentRuns.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getExperimentRuns.js","sourceRoot":"","sources":["../../../src/experiments/getExperimentRuns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,SAAS,MAAM,gBAAgB,CAAC;AAWvC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,MAAM,EAAE,OAAO,EACf,YAAY,GACY;IACxB,MAAM,MAAM,GAAG,OAAO,IAAI,YAAY,EAAE,CAAC;IACzC,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAC,sCAAsC,EAAE;QACxE,MAAM,EAAE;YACN,IAAI,EAAE;gBACJ,aAAa,EAAE,YAAY;aAC5B;SACF;KACF,CAAC,CAAC;IACH,MAAM,CAAC,qBAAqB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;IACpE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,EAAE,GAAG,qBAAqB,CAAC;IAC1E,SAAS,CAAC,kBAAkB,EAAE,oCAAoC,CAAC,CAAC;IACpE,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACnC,OAAO;gBACL,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,OAAO,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;gBAC7B,YAAY,EAAE,GAAG,CAAC,aAAa;gBAC/B,gBAAgB,EAAE,GAAG,CAAC,kBAAkB;gBACxC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;gBACnC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC/B,MAAM,EAAE,GAAG,CAAC,MAAiC;gBAC7C,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI;aACzB,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/experiments/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/experiments/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/experiments/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/experiments/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
2
|
+
import { HeadersOptions } from "openapi-fetch";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a provider that exports traces to Phoenix.
|
|
5
|
+
*/
|
|
6
|
+
export declare function createProvider({ projectName, baseUrl, headers, }: {
|
|
7
|
+
projectName: string;
|
|
8
|
+
headers: HeadersOptions;
|
|
9
|
+
/**
|
|
10
|
+
* The base URL of the Phoenix. Doesn't include the /v1/traces path.
|
|
11
|
+
*/
|
|
12
|
+
baseUrl: string;
|
|
13
|
+
}): NodeTracerProvider;
|
|
14
|
+
/**
|
|
15
|
+
* For dry runs we create a provider that doesn't export traces.
|
|
16
|
+
*/
|
|
17
|
+
export declare function createNoOpProvider(): NodeTracerProvider;
|
|
18
|
+
//# sourceMappingURL=instrumention.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"instrumention.d.ts","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,GACR,EAAE;IACD,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB,sBAoBA;AAED;;GAEG;AACH,wBAAgB,kBAAkB,uBAIjC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
|
|
2
|
+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
|
|
3
|
+
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
4
|
+
import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
|
|
5
|
+
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
6
|
+
import { SEMRESATTRS_PROJECT_NAME } from "@arizeai/openinference-semantic-conventions";
|
|
7
|
+
/**
|
|
8
|
+
* Creates a provider that exports traces to Phoenix.
|
|
9
|
+
*/
|
|
10
|
+
export function createProvider({ projectName, baseUrl, headers, }) {
|
|
11
|
+
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR);
|
|
12
|
+
const provider = new NodeTracerProvider({
|
|
13
|
+
resource: resourceFromAttributes({
|
|
14
|
+
[SEMRESATTRS_PROJECT_NAME]: projectName,
|
|
15
|
+
}),
|
|
16
|
+
spanProcessors: [
|
|
17
|
+
new SimpleSpanProcessor(new OTLPTraceExporter({
|
|
18
|
+
url: `${baseUrl}/v1/traces`,
|
|
19
|
+
headers: Array.isArray(headers)
|
|
20
|
+
? Object.fromEntries(headers)
|
|
21
|
+
: headers,
|
|
22
|
+
})),
|
|
23
|
+
],
|
|
24
|
+
});
|
|
25
|
+
return provider;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* For dry runs we create a provider that doesn't export traces.
|
|
29
|
+
*/
|
|
30
|
+
export function createNoOpProvider() {
|
|
31
|
+
const provider = new NodeTracerProvider({});
|
|
32
|
+
return provider;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=instrumention.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"instrumention.js","sourceRoot":"","sources":["../../../src/experiments/instrumention.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,MAAM,6CAA6C,CAAC;AAGvF;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,EAC7B,WAAW,EACX,OAAO,EACP,OAAO,GAQR;IACC,IAAI,CAAC,SAAS,CAAC,IAAI,iBAAiB,EAAE,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;QACtC,QAAQ,EAAE,sBAAsB,CAAC;YAC/B,CAAC,wBAAwB,CAAC,EAAE,WAAW;SACxC,CAAC;QACF,cAAc,EAAE;YACd,IAAI,mBAAmB,CACrB,IAAI,iBAAiB,CAAC;gBACpB,GAAG,EAAE,GAAG,OAAO,YAAY;gBAC3B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;oBAC7B,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC;oBAC7B,CAAC,CAAC,OAAO;aACZ,CAAC,CACH;SACF;KACF,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAE5C,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type PhoenixClient } from "../client.js";
|
|
2
2
|
import { ClientFn } from "../types/core.js";
|
|
3
|
-
import {
|
|
3
|
+
import { DatasetSelector } from "../types/datasets.js";
|
|
4
4
|
import type { Evaluator, ExperimentTask, RanExperiment } from "../types/experiments.js";
|
|
5
5
|
import { type Logger } from "../types/logger.js";
|
|
6
6
|
import { AnnotatorKind } from "../types/annotations.js";
|
|
@@ -21,12 +21,13 @@ export type RunExperimentParams = ClientFn & {
|
|
|
21
21
|
experimentDescription?: string;
|
|
22
22
|
/**
|
|
23
23
|
* Experiment metadata
|
|
24
|
+
* E.x. modelName
|
|
24
25
|
*/
|
|
25
26
|
experimentMetadata?: Record<string, unknown>;
|
|
26
27
|
/**
|
|
27
28
|
* The dataset to run the experiment on
|
|
28
29
|
*/
|
|
29
|
-
dataset:
|
|
30
|
+
dataset: DatasetSelector;
|
|
30
31
|
/**
|
|
31
32
|
* The task to run
|
|
32
33
|
*/
|
|
@@ -35,10 +36,6 @@ export type RunExperimentParams = ClientFn & {
|
|
|
35
36
|
* The evaluators to use
|
|
36
37
|
*/
|
|
37
38
|
evaluators?: Evaluator[];
|
|
38
|
-
/**
|
|
39
|
-
* The project under which the experiment task traces are recorded
|
|
40
|
-
*/
|
|
41
|
-
projectName?: string;
|
|
42
39
|
/**
|
|
43
40
|
* The logger to use
|
|
44
41
|
*/
|
|
@@ -58,7 +55,23 @@ export type RunExperimentParams = ClientFn & {
|
|
|
58
55
|
dryRun?: number | boolean;
|
|
59
56
|
};
|
|
60
57
|
/**
|
|
61
|
-
*
|
|
58
|
+
* Runs an experiment using a given set of dataset of examples.
|
|
59
|
+
*
|
|
60
|
+
* An experiment is a user-defined task that runs on each example in a dataset. The results from
|
|
61
|
+
* each experiment can be evaluated using any number of evaluators to measure the behavior of the
|
|
62
|
+
* task. The experiment and evaluation results are stored in the Phoenix database for comparison
|
|
63
|
+
* and analysis.
|
|
64
|
+
*
|
|
65
|
+
* A `task` is either a sync or async function that returns a JSON serializable
|
|
66
|
+
* output. If the `task` is a function of one argument then that argument will be bound to the
|
|
67
|
+
* `input` field of the dataset example. Alternatively, the `task` can be a function of any
|
|
68
|
+
* combination of specific argument names that will be bound to special values:
|
|
69
|
+
*
|
|
70
|
+
* - `input`: The input field of the dataset example
|
|
71
|
+
* - `expected`: The expected or reference output of the dataset example
|
|
72
|
+
* - `reference`: An alias for `expected`
|
|
73
|
+
* - `metadata`: Metadata associated with the dataset example
|
|
74
|
+
* - `example`: The dataset `Example` object with all associated fields
|
|
62
75
|
*
|
|
63
76
|
* @example
|
|
64
77
|
* ```ts
|
|
@@ -68,14 +81,12 @@ export type RunExperimentParams = ClientFn & {
|
|
|
68
81
|
* dataset: "my-dataset",
|
|
69
82
|
* task: async (example) => example.input,
|
|
70
83
|
* evaluators: [
|
|
71
|
-
* asEvaluator("my-evaluator", "CODE", async (params) => params.output),
|
|
84
|
+
* asEvaluator({ name: "my-evaluator", kind: "CODE", evaluate: async (params) => params.output }),
|
|
72
85
|
* ],
|
|
73
86
|
* });
|
|
74
87
|
* ```
|
|
75
|
-
*
|
|
76
|
-
* @experimental This feature is not complete, and will change in the future.
|
|
77
88
|
*/
|
|
78
|
-
export declare function runExperiment({ experimentName
|
|
89
|
+
export declare function runExperiment({ experimentName, experimentDescription, experimentMetadata, client: _client, dataset: DatasetSelector, task, evaluators, logger, record, concurrency, dryRun, }: RunExperimentParams): Promise<RanExperiment>;
|
|
79
90
|
/**
|
|
80
91
|
* Evaluate an experiment.
|
|
81
92
|
*
|
|
@@ -84,7 +95,6 @@ export declare function runExperiment({ experimentName: _experimentName, experim
|
|
|
84
95
|
export declare function evaluateExperiment({ experiment, evaluators, client: _client, logger, concurrency, dryRun, }: {
|
|
85
96
|
/**
|
|
86
97
|
* The experiment to evaluate
|
|
87
|
-
* @todo also accept Experiment, and attempt to fetch the runs from the server
|
|
88
98
|
**/
|
|
89
99
|
experiment: RanExperiment;
|
|
90
100
|
/** The evaluators to use */
|
|
@@ -92,9 +102,9 @@ export declare function evaluateExperiment({ experiment, evaluators, client: _cl
|
|
|
92
102
|
/** The client to use */
|
|
93
103
|
client?: PhoenixClient;
|
|
94
104
|
/** The logger to use */
|
|
95
|
-
logger
|
|
105
|
+
logger?: Logger;
|
|
96
106
|
/** The number of evaluators to run in parallel */
|
|
97
|
-
concurrency
|
|
107
|
+
concurrency?: number;
|
|
98
108
|
/**
|
|
99
109
|
* Whether to run the evaluation as a dry run
|
|
100
110
|
* If a number is provided, the evaluation will be run for the first n runs
|
|
@@ -116,11 +126,4 @@ export declare function asEvaluator({ name, kind, evaluate, }: {
|
|
|
116
126
|
kind: AnnotatorKind;
|
|
117
127
|
evaluate: Evaluator["evaluate"];
|
|
118
128
|
}): Evaluator;
|
|
119
|
-
/**
|
|
120
|
-
* Generate a unique id.
|
|
121
|
-
*
|
|
122
|
-
* @deprecated Use id generated by phoenix instead.
|
|
123
|
-
* @returns A unique id.
|
|
124
|
-
*/
|
|
125
|
-
export declare function id(): string;
|
|
126
129
|
//# sourceMappingURL=runExperiment.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runExperiment.d.ts","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,
|
|
1
|
+
{"version":3,"file":"runExperiment.d.ts","sourceRoot":"","sources":["../../../src/experiments/runExperiment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAEL,eAAe,EAGhB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,SAAS,EAKT,cAAc,EACd,aAAa,EACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAI9C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAYrD;;;;GAIG;AACH,MAAM,MAAM,mBAAmB,GAAG,QAAQ,GAAG;IAC3C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7C;;OAEG;IACH,OAAO,EAAE,eAAe,CAAC;IACzB;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;OAEG;IACH,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAsB,aAAa,CAAC,EAClC,cAAc,EACd,qBAAqB,EACrB,kBAAuB,EACvB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,eAAe,EACxB,IAAI,EACJ,UAAU,EACV,MAAgB,EAChB,MAAa,EACb,WAAe,EACf,MAAc,GACf,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAoH9C;AAwHD;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,UAAU,EACV,MAAM,EAAE,OAAO,EACf,MAAgB,EAChB,WAAe,EACf,MAAc,GACf,EAAE;IACD;;QAEI;IACJ,UAAU,EAAE,aAAa,CAAC;IAC1B,4BAA4B;IAC5B,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,wBAAwB;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;SAIK;IACL,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CAC3B,GAAG,OAAO,CAAC,aAAa,CAAC,CAgKzB;AA8DD;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,EAC1B,IAAI,EACJ,IAAI,EACJ,QAAQ,GACT,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,aAAa,CAAC;IACpB,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;CACjC,GAAG,SAAS,CAMZ"}
|