@arizeai/phoenix-client 6.10.1 → 6.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -0
- package/dist/esm/__generated__/api/v1.d.ts +254 -2
- package/dist/esm/__generated__/api/v1.d.ts.map +1 -1
- package/dist/esm/jest/index.d.ts +5 -0
- package/dist/esm/jest/index.d.ts.map +1 -0
- package/dist/esm/jest/index.js +49 -0
- package/dist/esm/jest/index.js.map +1 -0
- package/dist/esm/jest/reporter.d.ts +13 -0
- package/dist/esm/jest/reporter.d.ts.map +1 -0
- package/dist/esm/jest/reporter.js +19 -0
- package/dist/esm/jest/reporter.js.map +1 -0
- package/dist/esm/prompts/sdks/toAI.d.ts +2 -2
- package/dist/esm/prompts/sdks/toAI.d.ts.map +1 -1
- package/dist/esm/prompts/sdks/toAI.js.map +1 -1
- package/dist/esm/prompts/sdks/toAnthropic.d.ts +2 -2
- package/dist/esm/prompts/sdks/toAnthropic.d.ts.map +1 -1
- package/dist/esm/prompts/sdks/toAnthropic.js.map +1 -1
- package/dist/esm/prompts/sdks/toOpenAI.d.ts +2 -2
- package/dist/esm/prompts/sdks/toOpenAI.d.ts.map +1 -1
- package/dist/esm/prompts/sdks/toOpenAI.js.map +1 -1
- package/dist/esm/prompts/sdks/toSDK.d.ts +8 -8
- package/dist/esm/prompts/sdks/toSDK.d.ts.map +1 -1
- package/dist/esm/prompts/sdks/toSDK.js.map +1 -1
- package/dist/esm/prompts/sdks/types.d.ts +2 -2
- package/dist/esm/prompts/sdks/types.d.ts.map +1 -1
- package/dist/esm/schemas/llm/anthropic/converters.d.ts +8 -8
- package/dist/esm/schemas/llm/anthropic/messagePartSchemas.d.ts +4 -4
- package/dist/esm/schemas/llm/anthropic/messageSchemas.d.ts +6 -6
- package/dist/esm/schemas/llm/constants.d.ts +3 -3
- package/dist/esm/schemas/llm/converters.d.ts +12 -12
- package/dist/esm/schemas/llm/openai/converters.d.ts +3 -3
- package/dist/esm/schemas/llm/schemas.d.ts +2 -2
- package/dist/esm/testing/acceptance.d.ts +20 -0
- package/dist/esm/testing/acceptance.d.ts.map +1 -0
- package/dist/esm/testing/acceptance.js +129 -0
- package/dist/esm/testing/acceptance.js.map +1 -0
- package/dist/esm/testing/define-api.d.ts +157 -0
- package/dist/esm/testing/define-api.d.ts.map +1 -0
- package/dist/esm/testing/define-api.js +78 -0
- package/dist/esm/testing/define-api.js.map +1 -0
- package/dist/esm/testing/helpers.d.ts +55 -0
- package/dist/esm/testing/helpers.d.ts.map +1 -0
- package/dist/esm/testing/helpers.js +179 -0
- package/dist/esm/testing/helpers.js.map +1 -0
- package/dist/esm/testing/phoenix-test-tracking.d.ts +68 -0
- package/dist/esm/testing/phoenix-test-tracking.d.ts.map +1 -0
- package/dist/esm/testing/phoenix-test-tracking.js +521 -0
- package/dist/esm/testing/phoenix-test-tracking.js.map +1 -0
- package/dist/esm/testing/report-artifacts.d.ts +45 -0
- package/dist/esm/testing/report-artifacts.d.ts.map +1 -0
- package/dist/esm/testing/report-artifacts.js +218 -0
- package/dist/esm/testing/report-artifacts.js.map +1 -0
- package/dist/esm/testing/report-run.d.ts +22 -0
- package/dist/esm/testing/report-run.d.ts.map +1 -0
- package/dist/esm/testing/report-run.js +41 -0
- package/dist/esm/testing/report-run.js.map +1 -0
- package/dist/esm/testing/reporter-format.d.ts +83 -0
- package/dist/esm/testing/reporter-format.d.ts.map +1 -0
- package/dist/esm/testing/reporter-format.js +852 -0
- package/dist/esm/testing/reporter-format.js.map +1 -0
- package/dist/esm/testing/runner.d.ts +31 -0
- package/dist/esm/testing/runner.d.ts.map +1 -0
- package/dist/esm/testing/runner.js +238 -0
- package/dist/esm/testing/runner.js.map +1 -0
- package/dist/esm/testing/state.d.ts +138 -0
- package/dist/esm/testing/state.d.ts.map +1 -0
- package/dist/esm/testing/state.js +31 -0
- package/dist/esm/testing/state.js.map +1 -0
- package/dist/esm/testing/types.d.ts +319 -0
- package/dist/esm/testing/types.d.ts.map +1 -0
- package/dist/esm/testing/types.js +9 -0
- package/dist/esm/testing/types.js.map +1 -0
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/esm/utils/channel.d.ts +7 -7
- package/dist/esm/utils/channel.d.ts.map +1 -1
- package/dist/esm/utils/channel.js +1 -1
- package/dist/esm/utils/channel.js.map +1 -1
- package/dist/esm/utils/formatPromptMessages.d.ts.map +1 -1
- package/dist/esm/utils/getPromptBySelector.d.ts.map +1 -1
- package/dist/esm/utils/promisifyResult.d.ts +1 -1
- package/dist/esm/utils/promisifyResult.d.ts.map +1 -1
- package/dist/esm/utils/promisifyResult.js.map +1 -1
- package/dist/esm/utils/schemaMatches.d.ts +5 -5
- package/dist/esm/utils/schemaMatches.d.ts.map +1 -1
- package/dist/esm/utils/schemaMatches.js.map +1 -1
- package/dist/esm/vitest/index.d.ts +5 -0
- package/dist/esm/vitest/index.d.ts.map +1 -0
- package/dist/esm/vitest/index.js +15 -0
- package/dist/esm/vitest/index.js.map +1 -0
- package/dist/esm/vitest/reporter.d.ts +19 -0
- package/dist/esm/vitest/reporter.d.ts.map +1 -0
- package/dist/esm/vitest/reporter.js +27 -0
- package/dist/esm/vitest/reporter.js.map +1 -0
- package/dist/src/__generated__/api/v1.d.ts +254 -2
- package/dist/src/__generated__/api/v1.d.ts.map +1 -1
- package/dist/src/jest/index.d.ts +5 -0
- package/dist/src/jest/index.d.ts.map +1 -0
- package/dist/src/jest/index.js +58 -0
- package/dist/src/jest/index.js.map +1 -0
- package/dist/src/jest/reporter.d.ts +13 -0
- package/dist/src/jest/reporter.d.ts.map +1 -0
- package/dist/src/jest/reporter.js +23 -0
- package/dist/src/jest/reporter.js.map +1 -0
- package/dist/src/prompts/sdks/toAI.d.ts +2 -2
- package/dist/src/prompts/sdks/toAI.d.ts.map +1 -1
- package/dist/src/prompts/sdks/toAI.js.map +1 -1
- package/dist/src/prompts/sdks/toAnthropic.d.ts +2 -2
- package/dist/src/prompts/sdks/toAnthropic.d.ts.map +1 -1
- package/dist/src/prompts/sdks/toAnthropic.js.map +1 -1
- package/dist/src/prompts/sdks/toOpenAI.d.ts +2 -2
- package/dist/src/prompts/sdks/toOpenAI.d.ts.map +1 -1
- package/dist/src/prompts/sdks/toOpenAI.js.map +1 -1
- package/dist/src/prompts/sdks/toSDK.d.ts +8 -8
- package/dist/src/prompts/sdks/toSDK.d.ts.map +1 -1
- package/dist/src/prompts/sdks/toSDK.js.map +1 -1
- package/dist/src/prompts/sdks/types.d.ts +2 -2
- package/dist/src/prompts/sdks/types.d.ts.map +1 -1
- package/dist/src/schemas/llm/anthropic/converters.d.ts +8 -8
- package/dist/src/schemas/llm/anthropic/messagePartSchemas.d.ts +4 -4
- package/dist/src/schemas/llm/anthropic/messageSchemas.d.ts +6 -6
- package/dist/src/schemas/llm/constants.d.ts +3 -3
- package/dist/src/schemas/llm/converters.d.ts +12 -12
- package/dist/src/schemas/llm/openai/converters.d.ts +3 -3
- package/dist/src/schemas/llm/schemas.d.ts +2 -2
- package/dist/src/testing/acceptance.d.ts +20 -0
- package/dist/src/testing/acceptance.d.ts.map +1 -0
- package/dist/src/testing/acceptance.js +114 -0
- package/dist/src/testing/acceptance.js.map +1 -0
- package/dist/src/testing/define-api.d.ts +157 -0
- package/dist/src/testing/define-api.d.ts.map +1 -0
- package/dist/src/testing/define-api.js +81 -0
- package/dist/src/testing/define-api.js.map +1 -0
- package/dist/src/testing/helpers.d.ts +55 -0
- package/dist/src/testing/helpers.d.ts.map +1 -0
- package/dist/src/testing/helpers.js +182 -0
- package/dist/src/testing/helpers.js.map +1 -0
- package/dist/src/testing/phoenix-test-tracking.d.ts +68 -0
- package/dist/src/testing/phoenix-test-tracking.d.ts.map +1 -0
- package/dist/src/testing/phoenix-test-tracking.js +530 -0
- package/dist/src/testing/phoenix-test-tracking.js.map +1 -0
- package/dist/src/testing/report-artifacts.d.ts +45 -0
- package/dist/src/testing/report-artifacts.d.ts.map +1 -0
- package/dist/src/testing/report-artifacts.js +225 -0
- package/dist/src/testing/report-artifacts.js.map +1 -0
- package/dist/src/testing/report-run.d.ts +22 -0
- package/dist/src/testing/report-run.d.ts.map +1 -0
- package/dist/src/testing/report-run.js +47 -0
- package/dist/src/testing/report-run.js.map +1 -0
- package/dist/src/testing/reporter-format.d.ts +83 -0
- package/dist/src/testing/reporter-format.d.ts.map +1 -0
- package/dist/src/testing/reporter-format.js +870 -0
- package/dist/src/testing/reporter-format.js.map +1 -0
- package/dist/src/testing/runner.d.ts +31 -0
- package/dist/src/testing/runner.d.ts.map +1 -0
- package/dist/src/testing/runner.js +258 -0
- package/dist/src/testing/runner.js.map +1 -0
- package/dist/src/testing/state.d.ts +138 -0
- package/dist/src/testing/state.d.ts.map +1 -0
- package/dist/src/testing/state.js +38 -0
- package/dist/src/testing/state.js.map +1 -0
- package/dist/src/testing/types.d.ts +319 -0
- package/dist/src/testing/types.d.ts.map +1 -0
- package/dist/src/testing/types.js +13 -0
- package/dist/src/testing/types.js.map +1 -0
- package/dist/src/utils/channel.d.ts +7 -7
- package/dist/src/utils/channel.d.ts.map +1 -1
- package/dist/src/utils/channel.js +1 -1
- package/dist/src/utils/channel.js.map +1 -1
- package/dist/src/utils/formatPromptMessages.d.ts.map +1 -1
- package/dist/src/utils/getPromptBySelector.d.ts.map +1 -1
- package/dist/src/utils/promisifyResult.d.ts +1 -1
- package/dist/src/utils/promisifyResult.d.ts.map +1 -1
- package/dist/src/utils/promisifyResult.js.map +1 -1
- package/dist/src/utils/schemaMatches.d.ts +5 -5
- package/dist/src/utils/schemaMatches.d.ts.map +1 -1
- package/dist/src/utils/schemaMatches.js.map +1 -1
- package/dist/src/vitest/index.d.ts +5 -0
- package/dist/src/vitest/index.d.ts.map +1 -0
- package/dist/src/vitest/index.js +23 -0
- package/dist/src/vitest/index.js.map +1 -0
- package/dist/src/vitest/reporter.d.ts +19 -0
- package/dist/src/vitest/reporter.d.ts.map +1 -0
- package/dist/src/vitest/reporter.js +34 -0
- package/dist/src/vitest/reporter.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/ci-evals-annotations.mdx +190 -0
- package/docs/ci-evals-jest.mdx +78 -0
- package/docs/ci-evals-vitest.mdx +240 -0
- package/docs/ci-evals.mdx +263 -0
- package/docs/overview.mdx +9 -1
- package/package.json +49 -17
- package/src/__generated__/api/v1.ts +254 -2
- package/src/jest/index.ts +124 -0
- package/src/jest/reporter.ts +22 -0
- package/src/prompts/sdks/toAI.ts +4 -3
- package/src/prompts/sdks/toAnthropic.ts +4 -3
- package/src/prompts/sdks/toOpenAI.ts +4 -3
- package/src/prompts/sdks/toSDK.ts +16 -11
- package/src/prompts/sdks/types.ts +2 -2
- package/src/testing/acceptance.ts +190 -0
- package/src/testing/define-api.ts +279 -0
- package/src/testing/helpers.ts +251 -0
- package/src/testing/phoenix-test-tracking.ts +637 -0
- package/src/testing/report-artifacts.ts +272 -0
- package/src/testing/report-run.ts +44 -0
- package/src/testing/reporter-format.ts +1072 -0
- package/src/testing/runner.ts +350 -0
- package/src/testing/state.ts +165 -0
- package/src/testing/types.ts +366 -0
- package/src/utils/channel.ts +17 -15
- package/src/utils/promisifyResult.ts +6 -4
- package/src/utils/schemaMatches.ts +12 -10
- package/src/vitest/index.ts +57 -0
- package/src/vitest/reporter.ts +32 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.logOutput = logOutput;
|
|
4
|
+
exports.logAnnotation = logAnnotation;
|
|
5
|
+
exports.evaluate = evaluate;
|
|
6
|
+
exports.traceEvaluator = traceEvaluator;
|
|
7
|
+
exports.flushAnnotations = flushAnnotations;
|
|
8
|
+
const phoenix_test_tracking_1 = require("./phoenix-test-tracking");
|
|
9
|
+
const state_1 = require("./state");
|
|
10
|
+
/**
|
|
11
|
+
* Log the output produced by the test for the current run.
|
|
12
|
+
*
|
|
13
|
+
* Calling this multiple times overwrites the previously recorded value.
|
|
14
|
+
* The argument can be any JSON-serializable value — typically an object
|
|
15
|
+
* matching the shape of the example's `expected` field.
|
|
16
|
+
*/
|
|
17
|
+
function logOutput(output) {
|
|
18
|
+
const run = (0, state_1.currentRun)();
|
|
19
|
+
if (!run) {
|
|
20
|
+
throw new Error("logOutput() must be called inside a Phoenix eval test body");
|
|
21
|
+
}
|
|
22
|
+
run.output = output;
|
|
23
|
+
run.outputSet = true;
|
|
24
|
+
(0, phoenix_test_tracking_1.endTaskSpanForRun)(run);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Record an annotation on the current run.
|
|
28
|
+
*
|
|
29
|
+
* Annotations are collected during the test and posted to Phoenix as
|
|
30
|
+
* experiment evaluations after the test completes. The `name` is the
|
|
31
|
+
* Phoenix evaluation name; `score`, `label`, and `explanation` map to
|
|
32
|
+
* the standard Phoenix `EvaluationResult` fields.
|
|
33
|
+
*
|
|
34
|
+
* The annotation name `"pass"` is reserved — Phoenix eval tests always write
|
|
35
|
+
* a `pass` annotation derived from the test's assertion outcome, so a
|
|
36
|
+
* user-supplied annotation with that name would race / overwrite the
|
|
37
|
+
* built-in one. Such calls are silently ignored.
|
|
38
|
+
*/
|
|
39
|
+
function logAnnotation(annotation) {
|
|
40
|
+
const run = (0, state_1.currentRun)();
|
|
41
|
+
if (!run) {
|
|
42
|
+
throw new Error("logAnnotation() must be called inside a Phoenix eval test body");
|
|
43
|
+
}
|
|
44
|
+
if (annotation.name === "pass")
|
|
45
|
+
return;
|
|
46
|
+
run.annotations.push(annotation);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Run an evaluator object against the current test run and record the result.
|
|
50
|
+
*
|
|
51
|
+
* The evaluator may come from `@arizeai/phoenix-evals.createEvaluator`,
|
|
52
|
+
* `asExperimentEvaluator`, or any plain object with `{ name, evaluate }`.
|
|
53
|
+
* When `params` is omitted, the current test's `input`, recorded `output`,
|
|
54
|
+
* `expected`, `metadata`, and task `traceId` are supplied.
|
|
55
|
+
*/
|
|
56
|
+
async function evaluate(evaluator, params) {
|
|
57
|
+
var _a;
|
|
58
|
+
const run = (0, state_1.currentRun)();
|
|
59
|
+
if (!run) {
|
|
60
|
+
return await evaluator.evaluate((params !== null && params !== void 0 ? params : {}));
|
|
61
|
+
}
|
|
62
|
+
if (!run.outputSet && !(params && "output" in params)) {
|
|
63
|
+
warnEvaluateBeforeOutput(run.suite, evaluator.name, run.testName);
|
|
64
|
+
}
|
|
65
|
+
const evaluatorParams = Object.assign({ input: run.params.input,
|
|
66
|
+
// `run.output` is only ever set together with `outputSet`, so it is already
|
|
67
|
+
// `undefined` until a value is recorded.
|
|
68
|
+
output: run.output, expected: run.params.expected, metadata: run.params.metadata, traceId: (_a = run.traceId) !== null && _a !== void 0 ? _a : null }, (params !== null && params !== void 0 ? params : {}));
|
|
69
|
+
const { result, traceId } = await (0, phoenix_test_tracking_1.runEvaluatorWithTracing)(run.suite, evaluator.name, evaluatorParams, (paramsToEvaluate) => evaluator.evaluate(paramsToEvaluate));
|
|
70
|
+
logAnnotation(toAnnotation({
|
|
71
|
+
name: evaluator.name,
|
|
72
|
+
kind: evaluator.kind,
|
|
73
|
+
result,
|
|
74
|
+
traceId,
|
|
75
|
+
}));
|
|
76
|
+
return result;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Trace an evaluator function so its execution shows up as a separate
|
|
80
|
+
* `EVALUATOR` span in Phoenix and any `{ name, score }`-shaped return
|
|
81
|
+
* value is automatically captured as an annotation on the current run.
|
|
82
|
+
*
|
|
83
|
+
* The annotation name defaults to the traced function's name, falling
|
|
84
|
+
* back to `"evaluator"`.
|
|
85
|
+
*/
|
|
86
|
+
function traceEvaluator(fn, options) {
|
|
87
|
+
var _a;
|
|
88
|
+
const evaluatorName = (_a = options === null || options === void 0 ? void 0 : options.name) !== null && _a !== void 0 ? _a : (fn.name && fn.name !== "" ? fn.name : "evaluator");
|
|
89
|
+
return async (params) => {
|
|
90
|
+
const run = (0, state_1.currentRun)();
|
|
91
|
+
if (!run) {
|
|
92
|
+
// outside a test context, just call the function plainly
|
|
93
|
+
return await fn(params);
|
|
94
|
+
}
|
|
95
|
+
const { result, traceId } = await (0, phoenix_test_tracking_1.runEvaluatorWithTracing)(run.suite, evaluatorName, params, fn);
|
|
96
|
+
if (isAnnotationShaped(result)) {
|
|
97
|
+
logAnnotation(Object.assign(Object.assign({}, result), { traceId }));
|
|
98
|
+
}
|
|
99
|
+
return result;
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Warn (at most once per suite) when an evaluator runs before any output was
|
|
104
|
+
* recorded and none was passed explicitly. Such an evaluator receives
|
|
105
|
+
* `output: undefined`, which silently scores against nothing — almost always a
|
|
106
|
+
* forgotten `logOutput()`. Harmless for evaluators that only read `input`.
|
|
107
|
+
*/
|
|
108
|
+
const warnedOutputSuites = new WeakSet();
|
|
109
|
+
function warnEvaluateBeforeOutput(suite, evaluatorName, testName) {
|
|
110
|
+
if (warnedOutputSuites.has(suite))
|
|
111
|
+
return;
|
|
112
|
+
warnedOutputSuites.add(suite);
|
|
113
|
+
// eslint-disable-next-line no-console
|
|
114
|
+
console.warn(`[@arizeai/phoenix-client] evaluate("${evaluatorName}") ran before ` +
|
|
115
|
+
`logOutput() on test "${testName}", so the evaluator received ` +
|
|
116
|
+
`output=undefined. Call logOutput(...) first, or pass { output } ` +
|
|
117
|
+
`explicitly. (Ignore if this evaluator only needs input.)`);
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Normalize an evaluator's return value into an {@link Annotation}. The value
|
|
121
|
+
* is already typed as an {@link EvaluationResult}, so we only dispatch on its
|
|
122
|
+
* runtime shape: a string becomes a `label`, a number/boolean/null becomes a
|
|
123
|
+
* `score`, and an object contributes its `score`/`label`/`explanation`/
|
|
124
|
+
* `metadata` directly.
|
|
125
|
+
*/
|
|
126
|
+
function toAnnotation({ name, kind, result, traceId, }) {
|
|
127
|
+
const annotatorKind = kind !== null && kind !== void 0 ? kind : "CODE";
|
|
128
|
+
if (typeof result === "string") {
|
|
129
|
+
return { name, label: result, annotatorKind, traceId };
|
|
130
|
+
}
|
|
131
|
+
if (typeof result === "number" ||
|
|
132
|
+
typeof result === "boolean" ||
|
|
133
|
+
result === null) {
|
|
134
|
+
return { name, score: result, annotatorKind, traceId };
|
|
135
|
+
}
|
|
136
|
+
if (typeof result === "object" && !Array.isArray(result)) {
|
|
137
|
+
const { score, label, explanation, metadata } = result;
|
|
138
|
+
return {
|
|
139
|
+
name,
|
|
140
|
+
score,
|
|
141
|
+
label,
|
|
142
|
+
explanation,
|
|
143
|
+
metadata,
|
|
144
|
+
annotatorKind,
|
|
145
|
+
traceId,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
return { name, annotatorKind, traceId };
|
|
149
|
+
}
|
|
150
|
+
function isAnnotationShaped(value) {
|
|
151
|
+
if (!value || typeof value !== "object")
|
|
152
|
+
return false;
|
|
153
|
+
const v = value;
|
|
154
|
+
if (typeof v.name !== "string")
|
|
155
|
+
return false;
|
|
156
|
+
if (v.score !== undefined &&
|
|
157
|
+
typeof v.score !== "number" &&
|
|
158
|
+
typeof v.score !== "boolean" &&
|
|
159
|
+
v.score !== null) {
|
|
160
|
+
return false;
|
|
161
|
+
}
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Internal: persist all collected annotations for the run.
|
|
166
|
+
*
|
|
167
|
+
* Phoenix's `experiment_evaluations` endpoint is keyed by
|
|
168
|
+
* `(experiment_run_id, name)` so two annotations with the same name on
|
|
169
|
+
* the same run race each other. We collapse duplicates by name (last
|
|
170
|
+
* wins) up front, which makes the final state deterministic; the
|
|
171
|
+
* remaining writes target distinct names, so they post in parallel.
|
|
172
|
+
*/
|
|
173
|
+
async function flushAnnotations(runId, annotations, suite) {
|
|
174
|
+
if (!annotations.length)
|
|
175
|
+
return;
|
|
176
|
+
const byName = new Map();
|
|
177
|
+
for (const annotation of annotations) {
|
|
178
|
+
byName.set(annotation.name, annotation);
|
|
179
|
+
}
|
|
180
|
+
await Promise.all(Array.from(byName.values(), (annotation) => (0, phoenix_test_tracking_1.postAnnotation)(suite, runId, annotation)));
|
|
181
|
+
}
|
|
182
|
+
//# sourceMappingURL=helpers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"helpers.js","sourceRoot":"","sources":["../../../src/testing/helpers.ts"],"names":[],"mappings":";;AAsBA,8BAUC;AAeD,sCASC;AAUD,4BA0CC;AAUD,wCAuBC;AA8FD,4CAeC;AA1PD,mEAIiC;AACjC,mCAAsD;AAUtD;;;;;;GAMG;AACH,SAAgB,SAAS,CAAC,MAAe;IACvC,MAAM,GAAG,GAAG,IAAA,kBAAU,GAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;IACD,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;IACpB,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC;IACrB,IAAA,yCAAiB,EAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,aAAa,CAAC,UAAsB;IAClD,MAAM,GAAG,GAAG,IAAA,kBAAU,GAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO;IACvC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,QAAQ,CAI5B,SAAoC,EACpC,MAAgC;;IAEhC,MAAM,GAAG,GAAG,IAAA,kBAAU,GAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,EAAE,CAAW,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,IAAI,MAAM,CAAC,EAAE,CAAC;QACtD,wBAAwB,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,eAAe,GAAG,gBACtB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK;QACvB,4EAA4E;QAC5E,yCAAyC;QACzC,MAAM,EAAE,GAAG,CAAC,MAAM,EAClB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAC7B,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAC7B,OAAO,EAAE,MAAA,GAAG,CAAC,OAAO,mCAAI,IAAI,IACzB,CAAC,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,EAAE,CAAC,CACG,CAAC;IAEvB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,+CAAuB,EACvD,GAAG,CAAC,KAAK,EACT,SAAS,CAAC,IAAI,EACd,eAAe,EACf,CAAC,gBAAgB,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAC3D,CAAC;IACF,aAAa,CACX,YAAY,CAAC;QACX,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,MAAM;QACN,OAAO;KACR,CAAC,CACH,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,cAAc,CAC5B,EAA2E,EAC3E,OAA2B;;IAE3B,MAAM,aAAa,GACjB,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,mCAAI,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACvE,OAAO,KAAK,EAAE,MAAuB,EAAE,EAAE;QACvC,MAAM,GAAG,GAAG,IAAA,kBAAU,GAAE,CAAC;QACzB,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,yDAAyD;YACzD,OAAO,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,+CAAuB,EACvD,GAAG,CAAC,KAAK,EACT,aAAa,EACb,MAAM,EACN,EAAE,CACH,CAAC;QACF,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,aAAa,iCAAM,MAAM,KAAE,OAAO,IAAG,CAAC;QACxC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,IAAI,OAAO,EAAc,CAAC;AACrD,SAAS,wBAAwB,CAC/B,KAAiB,EACjB,aAAqB,EACrB,QAAgB;IAEhB,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO;IAC1C,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9B,sCAAsC;IACtC,OAAO,CAAC,IAAI,CACV,uCAAuC,aAAa,gBAAgB;QAClE,wBAAwB,QAAQ,+BAA+B;QAC/D,kEAAkE;QAClE,0DAA0D,CAC7D,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,EACpB,IAAI,EACJ,IAAI,EACJ,MAAM,EACN,OAAO,GAMR;IACC,MAAM,aAAa,GAAG,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,MAAM,CAAC;IACrC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IACzD,CAAC;IACD,IACE,OAAO,MAAM,KAAK,QAAQ;QAC1B,OAAO,MAAM,KAAK,SAAS;QAC3B,MAAM,KAAK,IAAI,EACf,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IACzD,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACzD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAC3C,MAAgC,CAAC;QACnC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,KAAK;YACL,WAAW;YACX,QAAQ;YACR,aAAa;YACb,OAAO;SACR,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,CAAC,GAAG,KAA4C,CAAC;IACvD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC7C,IACE,CAAC,CAAC,KAAK,KAAK,SAAS;QACrB,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;QAC3B,OAAO,CAAC,CAAC,KAAK,KAAK,SAAS;QAC5B,CAAC,CAAC,KAAK,KAAK,IAAI,EAChB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CACpC,KAAyB,EACzB,WAAyB,EACzB,KAAiB;IAEjB,IAAI,CAAC,WAAW,CAAC,MAAM;QAAE,OAAO;IAChC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,EAAE,EAAE,CACzC,IAAA,sCAAc,EAAC,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,CACzC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { type GlobalTracerProviderRegistration, type NodeTracerProvider } from "@arizeai/phoenix-otel";
|
|
2
|
+
import { type RunState, type SuiteState } from "./state";
|
|
3
|
+
import type { Annotation, KVMap } from "./types";
|
|
4
|
+
/**
|
|
5
|
+
* Decide whether tests should sync to Phoenix.
|
|
6
|
+
*
|
|
7
|
+
* Tracing is enabled by default. It can be disabled globally by setting
|
|
8
|
+
* `PHOENIX_TEST_TRACKING=false`, or per suite via `SuiteConfig.dryRun`.
|
|
9
|
+
*/
|
|
10
|
+
export declare function isTrackingEnabled(suite?: SuiteState): {
|
|
11
|
+
enabled: boolean;
|
|
12
|
+
reason?: string;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Resolve the repetition count for a test: per-test value, else suite-level,
|
|
16
|
+
* else the `PHOENIX_TEST_REPETITIONS` env var, else `1`. Non-positive or
|
|
17
|
+
* non-finite values fall back to `1`.
|
|
18
|
+
*/
|
|
19
|
+
export declare function resolveRepetitions(perTest: number | undefined, suite: SuiteState): number;
|
|
20
|
+
/**
|
|
21
|
+
* Initialize the suite: upload the dataset, create the experiment, and
|
|
22
|
+
* register the OpenInference tracer.
|
|
23
|
+
*
|
|
24
|
+
* If tracing is disabled (no Phoenix env vars, or PHOENIX_TEST_TRACKING=false),
|
|
25
|
+
* this populates a no-op tracer and exits without making any network calls.
|
|
26
|
+
*/
|
|
27
|
+
export declare function initializeSuite(suite: SuiteState): Promise<void>;
|
|
28
|
+
/**
|
|
29
|
+
* Wrap the user's test body in an OpenInference task span and return the
|
|
30
|
+
* trace id so we can submit it with the experiment run.
|
|
31
|
+
*/
|
|
32
|
+
export declare function runTaskWithTracing<Result>(suite: SuiteState, testName: string, fn: () => Promise<Result>): Promise<{
|
|
33
|
+
traceId: string;
|
|
34
|
+
result: Result;
|
|
35
|
+
} | {
|
|
36
|
+
traceId: string;
|
|
37
|
+
error: Error;
|
|
38
|
+
isTaskError: boolean;
|
|
39
|
+
}>;
|
|
40
|
+
/**
|
|
41
|
+
* End the current run's task span. `logOutput()` calls this immediately so
|
|
42
|
+
* evaluator work that follows is not included in the task span duration.
|
|
43
|
+
*/
|
|
44
|
+
export declare function endTaskSpanForRun(run: RunState): void;
|
|
45
|
+
/**
|
|
46
|
+
* POST a single experiment run for one test case to Phoenix.
|
|
47
|
+
*
|
|
48
|
+
* Best-effort: failures are captured and surfaced via the test reporter but
|
|
49
|
+
* do not fail the test itself.
|
|
50
|
+
*/
|
|
51
|
+
export declare function postExperimentRun(suite: SuiteState, run: RunState): Promise<string | undefined>;
|
|
52
|
+
/**
|
|
53
|
+
* POST one annotation (an "experiment_evaluation") for a run.
|
|
54
|
+
*/
|
|
55
|
+
export declare function postAnnotation(suite: SuiteState, runId: string | undefined, annotation: Annotation): Promise<void>;
|
|
56
|
+
/** Run an evaluator in an OpenInference evaluator span. */
|
|
57
|
+
export declare function runEvaluatorWithTracing<EvaluatorParams extends KVMap, EvaluatorResult>(suite: SuiteState, name: string, params: EvaluatorParams, fn: (params: EvaluatorParams) => EvaluatorResult | Promise<EvaluatorResult>): Promise<{
|
|
58
|
+
result: EvaluatorResult;
|
|
59
|
+
traceId: string | null;
|
|
60
|
+
}>;
|
|
61
|
+
/**
|
|
62
|
+
* Tear down the tracer provider created for the suite, flushing any pending
|
|
63
|
+
* spans before the process exits.
|
|
64
|
+
*/
|
|
65
|
+
export declare function teardownSuite(suite: SuiteState): Promise<void>;
|
|
66
|
+
/** Re-export the ones the entrypoint files consume. */
|
|
67
|
+
export type { GlobalTracerProviderRegistration, NodeTracerProvider };
|
|
68
|
+
//# sourceMappingURL=phoenix-test-tracking.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"phoenix-test-tracking.d.ts","sourceRoot":"","sources":["../../../src/testing/phoenix-test-tracking.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,gCAAgC,EAErC,KAAK,kBAAkB,EAQxB,MAAM,uBAAuB,CAAC;AAa/B,OAAO,EAAc,KAAK,QAAQ,EAAE,KAAK,UAAU,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAOjD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,CAAC,EAAE,UAAU,GAAG;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAQA;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,KAAK,EAAE,UAAU,GAChB,MAAM,CAaR;AAoGD;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAgKtE;AAeD;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAC7C,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,MAAM,EAChB,EAAE,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,GACxB,OAAO,CACN;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAC1D,CAiCA;AAMD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI,CAErD;AAsDD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,UAAU,EACjB,GAAG,EAAE,QAAQ,GACZ,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAwC7B;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,UAAU,EAAE,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAgCf;AAED,2DAA2D;AAC3D,wBAAsB,uBAAuB,CAC3C,eAAe,SAAS,KAAK,EAC7B,eAAe,EAEf,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,eAAe,EACvB,EAAE,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,GAC1E,OAAO,CAAC;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAiC9D;AAED;;;GAGG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAYpE;AAgBD,uDAAuD;AACvD,YAAY,EAAE,gCAAgC,EAAE,kBAAkB,EAAE,CAAC"}
|