@arizeai/phoenix-client 6.10.1 → 6.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/README.md +62 -0
  2. package/dist/esm/__generated__/api/v1.d.ts +254 -2
  3. package/dist/esm/__generated__/api/v1.d.ts.map +1 -1
  4. package/dist/esm/jest/index.d.ts +5 -0
  5. package/dist/esm/jest/index.d.ts.map +1 -0
  6. package/dist/esm/jest/index.js +49 -0
  7. package/dist/esm/jest/index.js.map +1 -0
  8. package/dist/esm/jest/reporter.d.ts +13 -0
  9. package/dist/esm/jest/reporter.d.ts.map +1 -0
  10. package/dist/esm/jest/reporter.js +19 -0
  11. package/dist/esm/jest/reporter.js.map +1 -0
  12. package/dist/esm/prompts/sdks/toAI.d.ts +2 -2
  13. package/dist/esm/prompts/sdks/toAI.d.ts.map +1 -1
  14. package/dist/esm/prompts/sdks/toAI.js.map +1 -1
  15. package/dist/esm/prompts/sdks/toAnthropic.d.ts +2 -2
  16. package/dist/esm/prompts/sdks/toAnthropic.d.ts.map +1 -1
  17. package/dist/esm/prompts/sdks/toAnthropic.js.map +1 -1
  18. package/dist/esm/prompts/sdks/toOpenAI.d.ts +2 -2
  19. package/dist/esm/prompts/sdks/toOpenAI.d.ts.map +1 -1
  20. package/dist/esm/prompts/sdks/toOpenAI.js.map +1 -1
  21. package/dist/esm/prompts/sdks/toSDK.d.ts +8 -8
  22. package/dist/esm/prompts/sdks/toSDK.d.ts.map +1 -1
  23. package/dist/esm/prompts/sdks/toSDK.js.map +1 -1
  24. package/dist/esm/prompts/sdks/types.d.ts +2 -2
  25. package/dist/esm/prompts/sdks/types.d.ts.map +1 -1
  26. package/dist/esm/schemas/llm/anthropic/converters.d.ts +8 -8
  27. package/dist/esm/schemas/llm/anthropic/messagePartSchemas.d.ts +4 -4
  28. package/dist/esm/schemas/llm/anthropic/messageSchemas.d.ts +6 -6
  29. package/dist/esm/schemas/llm/constants.d.ts +3 -3
  30. package/dist/esm/schemas/llm/converters.d.ts +12 -12
  31. package/dist/esm/schemas/llm/openai/converters.d.ts +3 -3
  32. package/dist/esm/schemas/llm/schemas.d.ts +2 -2
  33. package/dist/esm/testing/acceptance.d.ts +20 -0
  34. package/dist/esm/testing/acceptance.d.ts.map +1 -0
  35. package/dist/esm/testing/acceptance.js +129 -0
  36. package/dist/esm/testing/acceptance.js.map +1 -0
  37. package/dist/esm/testing/define-api.d.ts +157 -0
  38. package/dist/esm/testing/define-api.d.ts.map +1 -0
  39. package/dist/esm/testing/define-api.js +78 -0
  40. package/dist/esm/testing/define-api.js.map +1 -0
  41. package/dist/esm/testing/helpers.d.ts +55 -0
  42. package/dist/esm/testing/helpers.d.ts.map +1 -0
  43. package/dist/esm/testing/helpers.js +179 -0
  44. package/dist/esm/testing/helpers.js.map +1 -0
  45. package/dist/esm/testing/phoenix-test-tracking.d.ts +68 -0
  46. package/dist/esm/testing/phoenix-test-tracking.d.ts.map +1 -0
  47. package/dist/esm/testing/phoenix-test-tracking.js +521 -0
  48. package/dist/esm/testing/phoenix-test-tracking.js.map +1 -0
  49. package/dist/esm/testing/report-artifacts.d.ts +45 -0
  50. package/dist/esm/testing/report-artifacts.d.ts.map +1 -0
  51. package/dist/esm/testing/report-artifacts.js +218 -0
  52. package/dist/esm/testing/report-artifacts.js.map +1 -0
  53. package/dist/esm/testing/report-run.d.ts +22 -0
  54. package/dist/esm/testing/report-run.d.ts.map +1 -0
  55. package/dist/esm/testing/report-run.js +41 -0
  56. package/dist/esm/testing/report-run.js.map +1 -0
  57. package/dist/esm/testing/reporter-format.d.ts +83 -0
  58. package/dist/esm/testing/reporter-format.d.ts.map +1 -0
  59. package/dist/esm/testing/reporter-format.js +852 -0
  60. package/dist/esm/testing/reporter-format.js.map +1 -0
  61. package/dist/esm/testing/runner.d.ts +31 -0
  62. package/dist/esm/testing/runner.d.ts.map +1 -0
  63. package/dist/esm/testing/runner.js +238 -0
  64. package/dist/esm/testing/runner.js.map +1 -0
  65. package/dist/esm/testing/state.d.ts +138 -0
  66. package/dist/esm/testing/state.d.ts.map +1 -0
  67. package/dist/esm/testing/state.js +31 -0
  68. package/dist/esm/testing/state.js.map +1 -0
  69. package/dist/esm/testing/types.d.ts +319 -0
  70. package/dist/esm/testing/types.d.ts.map +1 -0
  71. package/dist/esm/testing/types.js +9 -0
  72. package/dist/esm/testing/types.js.map +1 -0
  73. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  74. package/dist/esm/utils/channel.d.ts +7 -7
  75. package/dist/esm/utils/channel.d.ts.map +1 -1
  76. package/dist/esm/utils/channel.js +1 -1
  77. package/dist/esm/utils/channel.js.map +1 -1
  78. package/dist/esm/utils/formatPromptMessages.d.ts.map +1 -1
  79. package/dist/esm/utils/getPromptBySelector.d.ts.map +1 -1
  80. package/dist/esm/utils/promisifyResult.d.ts +1 -1
  81. package/dist/esm/utils/promisifyResult.d.ts.map +1 -1
  82. package/dist/esm/utils/promisifyResult.js.map +1 -1
  83. package/dist/esm/utils/schemaMatches.d.ts +5 -5
  84. package/dist/esm/utils/schemaMatches.d.ts.map +1 -1
  85. package/dist/esm/utils/schemaMatches.js.map +1 -1
  86. package/dist/esm/vitest/index.d.ts +5 -0
  87. package/dist/esm/vitest/index.d.ts.map +1 -0
  88. package/dist/esm/vitest/index.js +15 -0
  89. package/dist/esm/vitest/index.js.map +1 -0
  90. package/dist/esm/vitest/reporter.d.ts +19 -0
  91. package/dist/esm/vitest/reporter.d.ts.map +1 -0
  92. package/dist/esm/vitest/reporter.js +27 -0
  93. package/dist/esm/vitest/reporter.js.map +1 -0
  94. package/dist/src/__generated__/api/v1.d.ts +254 -2
  95. package/dist/src/__generated__/api/v1.d.ts.map +1 -1
  96. package/dist/src/jest/index.d.ts +5 -0
  97. package/dist/src/jest/index.d.ts.map +1 -0
  98. package/dist/src/jest/index.js +58 -0
  99. package/dist/src/jest/index.js.map +1 -0
  100. package/dist/src/jest/reporter.d.ts +13 -0
  101. package/dist/src/jest/reporter.d.ts.map +1 -0
  102. package/dist/src/jest/reporter.js +23 -0
  103. package/dist/src/jest/reporter.js.map +1 -0
  104. package/dist/src/prompts/sdks/toAI.d.ts +2 -2
  105. package/dist/src/prompts/sdks/toAI.d.ts.map +1 -1
  106. package/dist/src/prompts/sdks/toAI.js.map +1 -1
  107. package/dist/src/prompts/sdks/toAnthropic.d.ts +2 -2
  108. package/dist/src/prompts/sdks/toAnthropic.d.ts.map +1 -1
  109. package/dist/src/prompts/sdks/toAnthropic.js.map +1 -1
  110. package/dist/src/prompts/sdks/toOpenAI.d.ts +2 -2
  111. package/dist/src/prompts/sdks/toOpenAI.d.ts.map +1 -1
  112. package/dist/src/prompts/sdks/toOpenAI.js.map +1 -1
  113. package/dist/src/prompts/sdks/toSDK.d.ts +8 -8
  114. package/dist/src/prompts/sdks/toSDK.d.ts.map +1 -1
  115. package/dist/src/prompts/sdks/toSDK.js.map +1 -1
  116. package/dist/src/prompts/sdks/types.d.ts +2 -2
  117. package/dist/src/prompts/sdks/types.d.ts.map +1 -1
  118. package/dist/src/schemas/llm/anthropic/converters.d.ts +8 -8
  119. package/dist/src/schemas/llm/anthropic/messagePartSchemas.d.ts +4 -4
  120. package/dist/src/schemas/llm/anthropic/messageSchemas.d.ts +6 -6
  121. package/dist/src/schemas/llm/constants.d.ts +3 -3
  122. package/dist/src/schemas/llm/converters.d.ts +12 -12
  123. package/dist/src/schemas/llm/openai/converters.d.ts +3 -3
  124. package/dist/src/schemas/llm/schemas.d.ts +2 -2
  125. package/dist/src/testing/acceptance.d.ts +20 -0
  126. package/dist/src/testing/acceptance.d.ts.map +1 -0
  127. package/dist/src/testing/acceptance.js +114 -0
  128. package/dist/src/testing/acceptance.js.map +1 -0
  129. package/dist/src/testing/define-api.d.ts +157 -0
  130. package/dist/src/testing/define-api.d.ts.map +1 -0
  131. package/dist/src/testing/define-api.js +81 -0
  132. package/dist/src/testing/define-api.js.map +1 -0
  133. package/dist/src/testing/helpers.d.ts +55 -0
  134. package/dist/src/testing/helpers.d.ts.map +1 -0
  135. package/dist/src/testing/helpers.js +182 -0
  136. package/dist/src/testing/helpers.js.map +1 -0
  137. package/dist/src/testing/phoenix-test-tracking.d.ts +68 -0
  138. package/dist/src/testing/phoenix-test-tracking.d.ts.map +1 -0
  139. package/dist/src/testing/phoenix-test-tracking.js +530 -0
  140. package/dist/src/testing/phoenix-test-tracking.js.map +1 -0
  141. package/dist/src/testing/report-artifacts.d.ts +45 -0
  142. package/dist/src/testing/report-artifacts.d.ts.map +1 -0
  143. package/dist/src/testing/report-artifacts.js +225 -0
  144. package/dist/src/testing/report-artifacts.js.map +1 -0
  145. package/dist/src/testing/report-run.d.ts +22 -0
  146. package/dist/src/testing/report-run.d.ts.map +1 -0
  147. package/dist/src/testing/report-run.js +47 -0
  148. package/dist/src/testing/report-run.js.map +1 -0
  149. package/dist/src/testing/reporter-format.d.ts +83 -0
  150. package/dist/src/testing/reporter-format.d.ts.map +1 -0
  151. package/dist/src/testing/reporter-format.js +870 -0
  152. package/dist/src/testing/reporter-format.js.map +1 -0
  153. package/dist/src/testing/runner.d.ts +31 -0
  154. package/dist/src/testing/runner.d.ts.map +1 -0
  155. package/dist/src/testing/runner.js +258 -0
  156. package/dist/src/testing/runner.js.map +1 -0
  157. package/dist/src/testing/state.d.ts +138 -0
  158. package/dist/src/testing/state.d.ts.map +1 -0
  159. package/dist/src/testing/state.js +38 -0
  160. package/dist/src/testing/state.js.map +1 -0
  161. package/dist/src/testing/types.d.ts +319 -0
  162. package/dist/src/testing/types.d.ts.map +1 -0
  163. package/dist/src/testing/types.js +13 -0
  164. package/dist/src/testing/types.js.map +1 -0
  165. package/dist/src/utils/channel.d.ts +7 -7
  166. package/dist/src/utils/channel.d.ts.map +1 -1
  167. package/dist/src/utils/channel.js +1 -1
  168. package/dist/src/utils/channel.js.map +1 -1
  169. package/dist/src/utils/formatPromptMessages.d.ts.map +1 -1
  170. package/dist/src/utils/getPromptBySelector.d.ts.map +1 -1
  171. package/dist/src/utils/promisifyResult.d.ts +1 -1
  172. package/dist/src/utils/promisifyResult.d.ts.map +1 -1
  173. package/dist/src/utils/promisifyResult.js.map +1 -1
  174. package/dist/src/utils/schemaMatches.d.ts +5 -5
  175. package/dist/src/utils/schemaMatches.d.ts.map +1 -1
  176. package/dist/src/utils/schemaMatches.js.map +1 -1
  177. package/dist/src/vitest/index.d.ts +5 -0
  178. package/dist/src/vitest/index.d.ts.map +1 -0
  179. package/dist/src/vitest/index.js +23 -0
  180. package/dist/src/vitest/index.js.map +1 -0
  181. package/dist/src/vitest/reporter.d.ts +19 -0
  182. package/dist/src/vitest/reporter.d.ts.map +1 -0
  183. package/dist/src/vitest/reporter.js +34 -0
  184. package/dist/src/vitest/reporter.js.map +1 -0
  185. package/dist/tsconfig.tsbuildinfo +1 -1
  186. package/docs/ci-evals-annotations.mdx +190 -0
  187. package/docs/ci-evals-jest.mdx +78 -0
  188. package/docs/ci-evals-vitest.mdx +240 -0
  189. package/docs/ci-evals.mdx +263 -0
  190. package/docs/overview.mdx +9 -1
  191. package/package.json +49 -17
  192. package/src/__generated__/api/v1.ts +254 -2
  193. package/src/jest/index.ts +124 -0
  194. package/src/jest/reporter.ts +22 -0
  195. package/src/prompts/sdks/toAI.ts +4 -3
  196. package/src/prompts/sdks/toAnthropic.ts +4 -3
  197. package/src/prompts/sdks/toOpenAI.ts +4 -3
  198. package/src/prompts/sdks/toSDK.ts +16 -11
  199. package/src/prompts/sdks/types.ts +2 -2
  200. package/src/testing/acceptance.ts +190 -0
  201. package/src/testing/define-api.ts +279 -0
  202. package/src/testing/helpers.ts +251 -0
  203. package/src/testing/phoenix-test-tracking.ts +637 -0
  204. package/src/testing/report-artifacts.ts +272 -0
  205. package/src/testing/report-run.ts +44 -0
  206. package/src/testing/reporter-format.ts +1072 -0
  207. package/src/testing/runner.ts +350 -0
  208. package/src/testing/state.ts +165 -0
  209. package/src/testing/types.ts +366 -0
  210. package/src/utils/channel.ts +17 -15
  211. package/src/utils/promisifyResult.ts +6 -4
  212. package/src/utils/schemaMatches.ts +12 -10
  213. package/src/vitest/index.ts +57 -0
  214. package/src/vitest/reporter.ts +32 -0
@@ -0,0 +1,182 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.logOutput = logOutput;
4
+ exports.logAnnotation = logAnnotation;
5
+ exports.evaluate = evaluate;
6
+ exports.traceEvaluator = traceEvaluator;
7
+ exports.flushAnnotations = flushAnnotations;
8
+ const phoenix_test_tracking_1 = require("./phoenix-test-tracking");
9
+ const state_1 = require("./state");
10
+ /**
11
+ * Log the output produced by the test for the current run.
12
+ *
13
+ * Calling this multiple times overwrites the previously recorded value.
14
+ * The argument can be any JSON-serializable value — typically an object
15
+ * matching the shape of the example's `expected` field.
16
+ */
17
+ function logOutput(output) {
18
+ const run = (0, state_1.currentRun)();
19
+ if (!run) {
20
+ throw new Error("logOutput() must be called inside a Phoenix eval test body");
21
+ }
22
+ run.output = output;
23
+ run.outputSet = true;
24
+ (0, phoenix_test_tracking_1.endTaskSpanForRun)(run);
25
+ }
26
+ /**
27
+ * Record an annotation on the current run.
28
+ *
29
+ * Annotations are collected during the test and posted to Phoenix as
30
+ * experiment evaluations after the test completes. The `name` is the
31
+ * Phoenix evaluation name; `score`, `label`, and `explanation` map to
32
+ * the standard Phoenix `EvaluationResult` fields.
33
+ *
34
+ * The annotation name `"pass"` is reserved — Phoenix eval tests always write
35
+ * a `pass` annotation derived from the test's assertion outcome, so a
36
+ * user-supplied annotation with that name would race / overwrite the
37
+ * built-in one. Such calls are silently ignored.
38
+ */
39
+ function logAnnotation(annotation) {
40
+ const run = (0, state_1.currentRun)();
41
+ if (!run) {
42
+ throw new Error("logAnnotation() must be called inside a Phoenix eval test body");
43
+ }
44
+ if (annotation.name === "pass")
45
+ return;
46
+ run.annotations.push(annotation);
47
+ }
48
+ /**
49
+ * Run an evaluator object against the current test run and record the result.
50
+ *
51
+ * The evaluator may come from `@arizeai/phoenix-evals.createEvaluator`,
52
+ * `asExperimentEvaluator`, or any plain object with `{ name, evaluate }`.
53
+ * When `params` is omitted, the current test's `input`, recorded `output`,
54
+ * `expected`, `metadata`, and task `traceId` are supplied.
55
+ */
56
+ async function evaluate(evaluator, params) {
57
+ var _a;
58
+ const run = (0, state_1.currentRun)();
59
+ if (!run) {
60
+ return await evaluator.evaluate((params !== null && params !== void 0 ? params : {}));
61
+ }
62
+ if (!run.outputSet && !(params && "output" in params)) {
63
+ warnEvaluateBeforeOutput(run.suite, evaluator.name, run.testName);
64
+ }
65
+ const evaluatorParams = Object.assign({ input: run.params.input,
66
+ // `run.output` is only ever set together with `outputSet`, so it is already
67
+ // `undefined` until a value is recorded.
68
+ output: run.output, expected: run.params.expected, metadata: run.params.metadata, traceId: (_a = run.traceId) !== null && _a !== void 0 ? _a : null }, (params !== null && params !== void 0 ? params : {}));
69
+ const { result, traceId } = await (0, phoenix_test_tracking_1.runEvaluatorWithTracing)(run.suite, evaluator.name, evaluatorParams, (paramsToEvaluate) => evaluator.evaluate(paramsToEvaluate));
70
+ logAnnotation(toAnnotation({
71
+ name: evaluator.name,
72
+ kind: evaluator.kind,
73
+ result,
74
+ traceId,
75
+ }));
76
+ return result;
77
+ }
78
+ /**
79
+ * Trace an evaluator function so its execution shows up as a separate
80
+ * `EVALUATOR` span in Phoenix and any `{ name, score }`-shaped return
81
+ * value is automatically captured as an annotation on the current run.
82
+ *
83
+ * The annotation name defaults to the traced function's name, falling
84
+ * back to `"evaluator"`.
85
+ */
86
+ function traceEvaluator(fn, options) {
87
+ var _a;
88
+ const evaluatorName = (_a = options === null || options === void 0 ? void 0 : options.name) !== null && _a !== void 0 ? _a : (fn.name && fn.name !== "" ? fn.name : "evaluator");
89
+ return async (params) => {
90
+ const run = (0, state_1.currentRun)();
91
+ if (!run) {
92
+ // outside a test context, just call the function plainly
93
+ return await fn(params);
94
+ }
95
+ const { result, traceId } = await (0, phoenix_test_tracking_1.runEvaluatorWithTracing)(run.suite, evaluatorName, params, fn);
96
+ if (isAnnotationShaped(result)) {
97
+ logAnnotation(Object.assign(Object.assign({}, result), { traceId }));
98
+ }
99
+ return result;
100
+ };
101
+ }
102
+ /**
103
+ * Warn (at most once per suite) when an evaluator runs before any output was
104
+ * recorded and none was passed explicitly. Such an evaluator receives
105
+ * `output: undefined`, which silently scores against nothing — almost always a
106
+ * forgotten `logOutput()`. Harmless for evaluators that only read `input`.
107
+ */
108
+ const warnedOutputSuites = new WeakSet();
109
+ function warnEvaluateBeforeOutput(suite, evaluatorName, testName) {
110
+ if (warnedOutputSuites.has(suite))
111
+ return;
112
+ warnedOutputSuites.add(suite);
113
+ // eslint-disable-next-line no-console
114
+ console.warn(`[@arizeai/phoenix-client] evaluate("${evaluatorName}") ran before ` +
115
+ `logOutput() on test "${testName}", so the evaluator received ` +
116
+ `output=undefined. Call logOutput(...) first, or pass { output } ` +
117
+ `explicitly. (Ignore if this evaluator only needs input.)`);
118
+ }
119
+ /**
120
+ * Normalize an evaluator's return value into an {@link Annotation}. The value
121
+ * is already typed as an {@link EvaluationResult}, so we only dispatch on its
122
+ * runtime shape: a string becomes a `label`, a number/boolean/null becomes a
123
+ * `score`, and an object contributes its `score`/`label`/`explanation`/
124
+ * `metadata` directly.
125
+ */
126
+ function toAnnotation({ name, kind, result, traceId, }) {
127
+ const annotatorKind = kind !== null && kind !== void 0 ? kind : "CODE";
128
+ if (typeof result === "string") {
129
+ return { name, label: result, annotatorKind, traceId };
130
+ }
131
+ if (typeof result === "number" ||
132
+ typeof result === "boolean" ||
133
+ result === null) {
134
+ return { name, score: result, annotatorKind, traceId };
135
+ }
136
+ if (typeof result === "object" && !Array.isArray(result)) {
137
+ const { score, label, explanation, metadata } = result;
138
+ return {
139
+ name,
140
+ score,
141
+ label,
142
+ explanation,
143
+ metadata,
144
+ annotatorKind,
145
+ traceId,
146
+ };
147
+ }
148
+ return { name, annotatorKind, traceId };
149
+ }
150
+ function isAnnotationShaped(value) {
151
+ if (!value || typeof value !== "object")
152
+ return false;
153
+ const v = value;
154
+ if (typeof v.name !== "string")
155
+ return false;
156
+ if (v.score !== undefined &&
157
+ typeof v.score !== "number" &&
158
+ typeof v.score !== "boolean" &&
159
+ v.score !== null) {
160
+ return false;
161
+ }
162
+ return true;
163
+ }
164
+ /**
165
+ * Internal: persist all collected annotations for the run.
166
+ *
167
+ * Phoenix's `experiment_evaluations` endpoint is keyed by
168
+ * `(experiment_run_id, name)` so two annotations with the same name on
169
+ * the same run race each other. We collapse duplicates by name (last
170
+ * wins) up front, which makes the final state deterministic; the
171
+ * remaining writes target distinct names, so they post in parallel.
172
+ */
173
+ async function flushAnnotations(runId, annotations, suite) {
174
+ if (!annotations.length)
175
+ return;
176
+ const byName = new Map();
177
+ for (const annotation of annotations) {
178
+ byName.set(annotation.name, annotation);
179
+ }
180
+ await Promise.all(Array.from(byName.values(), (annotation) => (0, phoenix_test_tracking_1.postAnnotation)(suite, runId, annotation)));
181
+ }
182
+ //# sourceMappingURL=helpers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"helpers.js","sourceRoot":"","sources":["../../../src/testing/helpers.ts"],"names":[],"mappings":";;AAsBA,8BAUC;AAeD,sCASC;AAUD,4BA0CC;AAUD,wCAuBC;AA8FD,4CAeC;AA1PD,mEAIiC;AACjC,mCAAsD;AAUtD;;;;;;GAMG;AACH,SAAgB,SAAS,CAAC,MAAe;IACvC,MAAM,GAAG,GAAG,IAAA,kBAAU,GAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;IACD,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;IACpB,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC;IACrB,IAAA,yCAAiB,EAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,aAAa,CAAC,UAAsB;IAClD,MAAM,GAAG,GAAG,IAAA,kBAAU,GAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO;IACvC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,QAAQ,CAI5B,SAAoC,EACpC,MAAgC;;IAEhC,MAAM,GAAG,GAAG,IAAA,kBAAU,GAAE,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,EAAE,CAAW,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,IAAI,MAAM,CAAC,EAAE,CAAC;QACtD,wBAAwB,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,eAAe,GAAG,gBACtB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK;QACvB,4EAA4E;QAC5E,yCAAyC;QACzC,MAAM,EAAE,GAAG,CAAC,MAAM,EAClB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAC7B,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAC7B,OAAO,EAAE,MAAA,GAAG,CAAC,OAAO,mCAAI,IAAI,IACzB,CAAC,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,EAAE,CAAC,CACG,CAAC;IAEvB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,+CAAuB,EACvD,GAAG,CAAC,KAAK,EACT,SAAS,CAAC,IAAI,EACd,eAAe,EACf,CAAC,gBAAgB,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAC3D,CAAC;IACF,aAAa,CACX,YAAY,CAAC;QACX,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,MAAM;QACN,OAAO;KACR,CAAC,CACH,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,cAAc,CAC5B,EAA2E,EAC3E,OAA2B;;IAE3B,MAAM,aAAa,GACjB,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,mCAAI,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACvE,OAAO,KAAK,EAAE,MAAuB,EAAE,EAAE;QACvC,MAAM,GAAG,GAAG,IAAA,kBAAU,GAAE,CAAC;QACzB,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,yDAAyD;YACzD,OAAO,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,+CAAuB,EACvD,GAAG,CAAC,KAAK,EACT,aAAa,EACb,MAAM,EACN,EAAE,CACH,CAAC;QACF,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,aAAa,iCAAM,MAAM,KAAE,OAAO,IAAG,CAAC;QACxC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,IAAI,OAAO,EAAc,CAAC;AACrD,SAAS,wBAAwB,CAC/B,KAAiB,EACjB,aAAqB,EACrB,QAAgB;IAEhB,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO;IAC1C,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9B,sCAAsC;IACtC,OAAO,CAAC,IAAI,CACV,uCAAuC,aAAa,gBAAgB;QAClE,wBAAwB,QAAQ,+BAA+B;QAC/D,kEAAkE;QAClE,0DAA0D,CAC7D,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,EACpB,IAAI,EACJ,IAAI,EACJ,MAAM,EACN,OAAO,GAMR;IACC,MAAM,aAAa,GAAG,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,MAAM,CAAC;IACrC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IACzD,CAAC;IACD,IACE,OAAO,MAAM,KAAK,QAAQ;QAC1B,OAAO,MAAM,KAAK,SAAS;QAC3B,MAAM,KAAK,IAAI,EACf,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IACzD,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACzD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAC3C,MAAgC,CAAC;QACnC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,KAAK;YACL,WAAW;YACX,QAAQ;YACR,aAAa;YACb,OAAO;SACR,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,CAAC,GAAG,KAA4C,CAAC;IACvD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC7C,IACE,CAAC,CAAC,KAAK,KAAK,SAAS;QACrB,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;QAC3B,OAAO,CAAC,CAAC,KAAK,KAAK,SAAS;QAC5B,CAAC,CAAC,KAAK,KAAK,IAAI,EAChB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CACpC,KAAyB,EACzB,WAAyB,EACzB,KAAiB;IAEjB,IAAI,CAAC,WAAW,CAAC,MAAM;QAAE,OAAO;IAChC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,EAAE,EAAE,CACzC,IAAA,sCAAc,EAAC,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,CACzC,CACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,68 @@
1
+ import { type GlobalTracerProviderRegistration, type NodeTracerProvider } from "@arizeai/phoenix-otel";
2
+ import { type RunState, type SuiteState } from "./state";
3
+ import type { Annotation, KVMap } from "./types";
4
+ /**
5
+ * Decide whether tests should sync to Phoenix.
6
+ *
7
+ * Tracing is enabled by default. It can be disabled globally by setting
8
+ * `PHOENIX_TEST_TRACKING=false`, or per suite via `SuiteConfig.dryRun`.
9
+ */
10
+ export declare function isTrackingEnabled(suite?: SuiteState): {
11
+ enabled: boolean;
12
+ reason?: string;
13
+ };
14
+ /**
15
+ * Resolve the repetition count for a test: per-test value, else suite-level,
16
+ * else the `PHOENIX_TEST_REPETITIONS` env var, else `1`. Non-positive or
17
+ * non-finite values fall back to `1`.
18
+ */
19
+ export declare function resolveRepetitions(perTest: number | undefined, suite: SuiteState): number;
20
+ /**
21
+ * Initialize the suite: upload the dataset, create the experiment, and
22
+ * register the OpenInference tracer.
23
+ *
24
+ * If tracing is disabled (no Phoenix env vars, or PHOENIX_TEST_TRACKING=false),
25
+ * this populates a no-op tracer and exits without making any network calls.
26
+ */
27
+ export declare function initializeSuite(suite: SuiteState): Promise<void>;
28
+ /**
29
+ * Wrap the user's test body in an OpenInference task span and return the
30
+ * trace id so we can submit it with the experiment run.
31
+ */
32
+ export declare function runTaskWithTracing<Result>(suite: SuiteState, testName: string, fn: () => Promise<Result>): Promise<{
33
+ traceId: string;
34
+ result: Result;
35
+ } | {
36
+ traceId: string;
37
+ error: Error;
38
+ isTaskError: boolean;
39
+ }>;
40
+ /**
41
+ * End the current run's task span. `logOutput()` calls this immediately so
42
+ * evaluator work that follows is not included in the task span duration.
43
+ */
44
+ export declare function endTaskSpanForRun(run: RunState): void;
45
+ /**
46
+ * POST a single experiment run for one test case to Phoenix.
47
+ *
48
+ * Best-effort: failures are captured and surfaced via the test reporter but
49
+ * do not fail the test itself.
50
+ */
51
+ export declare function postExperimentRun(suite: SuiteState, run: RunState): Promise<string | undefined>;
52
+ /**
53
+ * POST one annotation (an "experiment_evaluation") for a run.
54
+ */
55
+ export declare function postAnnotation(suite: SuiteState, runId: string | undefined, annotation: Annotation): Promise<void>;
56
+ /** Run an evaluator in an OpenInference evaluator span. */
57
+ export declare function runEvaluatorWithTracing<EvaluatorParams extends KVMap, EvaluatorResult>(suite: SuiteState, name: string, params: EvaluatorParams, fn: (params: EvaluatorParams) => EvaluatorResult | Promise<EvaluatorResult>): Promise<{
58
+ result: EvaluatorResult;
59
+ traceId: string | null;
60
+ }>;
61
+ /**
62
+ * Tear down the tracer provider created for the suite, flushing any pending
63
+ * spans before the process exits.
64
+ */
65
+ export declare function teardownSuite(suite: SuiteState): Promise<void>;
66
+ /** Re-export the ones the entrypoint files consume. */
67
+ export type { GlobalTracerProviderRegistration, NodeTracerProvider };
68
+ //# sourceMappingURL=phoenix-test-tracking.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"phoenix-test-tracking.d.ts","sourceRoot":"","sources":["../../../src/testing/phoenix-test-tracking.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,gCAAgC,EAErC,KAAK,kBAAkB,EAQxB,MAAM,uBAAuB,CAAC;AAa/B,OAAO,EAAc,KAAK,QAAQ,EAAE,KAAK,UAAU,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAOjD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,CAAC,EAAE,UAAU,GAAG;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAQA;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,KAAK,EAAE,UAAU,GAChB,MAAM,CAaR;AAoGD;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAgKtE;AAeD;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAC7C,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,MAAM,EAChB,EAAE,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,GACxB,OAAO,CACN;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAC1D,CAiCA;AAMD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI,CAErD;AAsDD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,UAAU,EACjB,GAAG,EAAE,QAAQ,GACZ,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAwC7B;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,UAAU,EAAE,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAgCf;AAED,2DAA2D;AAC3D,wBAAsB,uBAAuB,CAC3C,eAAe,SAAS,KAAK,EAC7B,eAAe,EAEf,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,eAAe,EACvB,EAAE,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,GAC1E,OAAO,CAAC;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAiC9D;AAED;;;GAGG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAYpE;AAgBD,uDAAuD;AACvD,YAAY,EAAE,gCAAgC,EAAE,kBAAkB,EAAE,CAAC"}