@arizeai/phoenix-evals 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/dist/esm/decision/decideClassification.d.ts +20 -0
  3. package/dist/esm/decision/decideClassification.d.ts.map +1 -0
  4. package/dist/esm/decision/decideClassification.js +72 -0
  5. package/dist/esm/decision/decideClassification.js.map +1 -0
  6. package/dist/esm/decision/decisionPrompt.d.ts +8 -0
  7. package/dist/esm/decision/decisionPrompt.d.ts.map +1 -0
  8. package/dist/esm/decision/decisionPrompt.js +66 -0
  9. package/dist/esm/decision/decisionPrompt.js.map +1 -0
  10. package/dist/esm/llm/ClassificationEvaluator.d.ts +2 -2
  11. package/dist/esm/llm/ClassificationEvaluator.d.ts.map +1 -1
  12. package/dist/esm/llm/ClassificationEvaluator.js.map +1 -1
  13. package/dist/esm/llm/generateClassification.d.ts +5 -5
  14. package/dist/esm/llm/generateClassification.d.ts.map +1 -1
  15. package/dist/esm/llm/generateClassification.js +20 -71
  16. package/dist/esm/llm/generateClassification.js.map +1 -1
  17. package/dist/esm/telemetry/index.d.ts +14 -0
  18. package/dist/esm/telemetry/index.d.ts.map +1 -1
  19. package/dist/esm/telemetry/index.js +15 -0
  20. package/dist/esm/telemetry/index.js.map +1 -1
  21. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  22. package/dist/esm/types/evals.d.ts +8 -6
  23. package/dist/esm/types/evals.d.ts.map +1 -1
  24. package/dist/esm/utils/index.d.ts +1 -0
  25. package/dist/esm/utils/index.d.ts.map +1 -1
  26. package/dist/esm/utils/index.js +1 -0
  27. package/dist/esm/utils/index.js.map +1 -1
  28. package/dist/esm/utils/isDecisionModel.d.ts +18 -0
  29. package/dist/esm/utils/isDecisionModel.d.ts.map +1 -0
  30. package/dist/esm/utils/isDecisionModel.js +13 -0
  31. package/dist/esm/utils/isDecisionModel.js.map +1 -0
  32. package/dist/src/decision/decideClassification.d.ts +20 -0
  33. package/dist/src/decision/decideClassification.d.ts.map +1 -0
  34. package/dist/src/decision/decideClassification.js +71 -0
  35. package/dist/src/decision/decideClassification.js.map +1 -0
  36. package/dist/src/decision/decisionPrompt.d.ts +8 -0
  37. package/dist/src/decision/decisionPrompt.d.ts.map +1 -0
  38. package/dist/src/decision/decisionPrompt.js +69 -0
  39. package/dist/src/decision/decisionPrompt.js.map +1 -0
  40. package/dist/src/llm/ClassificationEvaluator.d.ts +2 -2
  41. package/dist/src/llm/ClassificationEvaluator.d.ts.map +1 -1
  42. package/dist/src/llm/generateClassification.d.ts +5 -5
  43. package/dist/src/llm/generateClassification.d.ts.map +1 -1
  44. package/dist/src/llm/generateClassification.js +18 -70
  45. package/dist/src/llm/generateClassification.js.map +1 -1
  46. package/dist/src/telemetry/index.d.ts +14 -0
  47. package/dist/src/telemetry/index.d.ts.map +1 -1
  48. package/dist/src/telemetry/index.js +17 -0
  49. package/dist/src/telemetry/index.js.map +1 -1
  50. package/dist/src/types/evals.d.ts +8 -6
  51. package/dist/src/types/evals.d.ts.map +1 -1
  52. package/dist/src/utils/index.d.ts +1 -0
  53. package/dist/src/utils/index.d.ts.map +1 -1
  54. package/dist/src/utils/index.js +1 -0
  55. package/dist/src/utils/index.js.map +1 -1
  56. package/dist/src/utils/isDecisionModel.d.ts +18 -0
  57. package/dist/src/utils/isDecisionModel.d.ts.map +1 -0
  58. package/dist/src/utils/isDecisionModel.js +16 -0
  59. package/dist/src/utils/isDecisionModel.js.map +1 -0
  60. package/dist/tsconfig.tsbuildinfo +1 -1
  61. package/docs/classification.mdx +43 -1
  62. package/package.json +13 -13
  63. package/src/decision/decideClassification.ts +85 -0
  64. package/src/decision/decisionPrompt.ts +70 -0
  65. package/src/llm/ClassificationEvaluator.ts +2 -2
  66. package/src/llm/generateClassification.ts +20 -82
  67. package/src/telemetry/index.ts +21 -0
  68. package/src/types/evals.ts +8 -6
  69. package/src/utils/index.ts +1 -0
  70. package/src/utils/isDecisionModel.ts +29 -0
  71. package/dist/esm/utils/isEvaluationModel.d.ts +0 -13
  72. package/dist/esm/utils/isEvaluationModel.d.ts.map +0 -1
  73. package/dist/esm/utils/isEvaluationModel.js +0 -11
  74. package/dist/esm/utils/isEvaluationModel.js.map +0 -1
  75. package/dist/src/utils/isEvaluationModel.d.ts +0 -13
  76. package/dist/src/utils/isEvaluationModel.d.ts.map +0 -1
  77. package/dist/src/utils/isEvaluationModel.js +0 -14
  78. package/dist/src/utils/isEvaluationModel.js.map +0 -1
  79. package/src/utils/isEvaluationModel.ts +0 -23
package/CHANGELOG.md CHANGED
@@ -1,5 +1,15 @@
1
1
  # @arizeai/phoenix-evals
2
2
 
3
+ ## 2.7.0
4
+
5
+ ### Minor Changes
6
+
7
+ - b00c105: Classification evaluators, including every built-in, can now run on AI SDK decision models such as `openai.decisionModel("gpt-6-luna")`. The label comes from `experimental_decide`, with label probabilities (and the provider's confidence, when reported) in `metadata`. The rendered prompt is sent as plain text, and refusals throw the AI SDK's `Experimental_DecisionRefusalError`. Decide spans now record the decision state, question and answer. New exports: `isDecisionModel` and the `DecisionModel` type.
8
+
9
+ This replaces the evaluation model support from the previous release, and changes what evaluation models such as TypeSafe's Jev are sent. They now go through `experimental_decide` too. Message prompts are joined into plain text instead of being serialized as JSON state.
10
+
11
+ `ai` is pinned to `7.0.137` and `@ai-sdk/otel` to `1.0.137`, because the decision API is experimental and has changed in patch releases, and `@ai-sdk/otel` depends on one exact `ai` version. Use `@ai-sdk/openai` 4.0.89 or later with OpenAI decision models. Earlier versions receive the state as JSON-encoded text instead of plain text.
12
+
3
13
  ## 2.6.0
4
14
 
5
15
  ### Minor Changes
@@ -0,0 +1,20 @@
1
+ import type { ClassificationResult } from "../types/evals.js";
2
+ import type { TelemetryConfig } from "../types/otel.js";
3
+ import type { WithPrompt } from "../types/prompts.js";
4
+ import type { DecisionModel } from "../utils/isDecisionModel.js";
5
+ /**
6
+ * Classify with an AI SDK decision model by asking one choice question whose
7
+ * options are the labels. Decision models don't generate text, so the
8
+ * result has no explanation. Label probabilities, the provider's confidence
9
+ * (when it reports one) and the resolved model ID go in the metadata.
10
+ *
11
+ * Choice limits and refusals are left to the AI SDK and the provider: a
12
+ * refusal throws the AI SDK's `Experimental_DecisionRefusalError`.
13
+ */
14
+ export declare function decideClassification(args: {
15
+ model: DecisionModel;
16
+ labels: readonly string[];
17
+ prompt: WithPrompt;
18
+ telemetry?: TelemetryConfig;
19
+ }): Promise<ClassificationResult>;
20
+ //# sourceMappingURL=decideClassification.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"decideClassification.d.ts","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAe9D;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA2BhC"}
@@ -0,0 +1,72 @@
1
+ /**
2
+ * The only place phoenix-evals calls the AI SDK's experimental decision API.
3
+ * `experimental_decide` and the decision model spec may change in AI SDK
4
+ * patch releases, so keep everything that depends on their shape in here.
5
+ */
6
+ import { experimental_decide } from "ai";
7
+ import { getTelemetryOptions } from "../telemetry/index.js";
8
+ import { promptToDecisionState } from "./decisionPrompt.js";
9
+ /**
10
+ * The ID of the single choice question asked per classification.
11
+ */
12
+ const QUESTION_ID = "label";
13
+ /**
14
+ * The question's instructions. The rendered prompt, rubric included, is sent
15
+ * as the state.
16
+ */
17
+ const INSTRUCTIONS = "Read the prompt in the state and answer it by selecting exactly one of the choices.";
18
+ /**
19
+ * Classify with an AI SDK decision model by asking one choice question whose
20
+ * options are the labels. Decision models don't generate text, so the
21
+ * result has no explanation. Label probabilities, the provider's confidence
22
+ * (when it reports one) and the resolved model ID go in the metadata.
23
+ *
24
+ * Choice limits and refusals are left to the AI SDK and the provider: a
25
+ * refusal throws the AI SDK's `Experimental_DecisionRefusalError`.
26
+ */
27
+ export async function decideClassification(args) {
28
+ const { model, labels, prompt, telemetry } = args;
29
+ const result = await experimental_decide({
30
+ model,
31
+ state: promptToDecisionState(prompt),
32
+ questions: {
33
+ [QUESTION_ID]: {
34
+ type: "choice",
35
+ instructions: INSTRUCTIONS,
36
+ criteria: Object.fromEntries(labels.map((label) => [label, null])),
37
+ },
38
+ },
39
+ telemetry: getTelemetryOptions({
40
+ functionId: "decideClassification",
41
+ telemetry,
42
+ }),
43
+ });
44
+ const answer = result.answers[QUESTION_ID];
45
+ const confidence = getConfidence(result.providerMetadata);
46
+ return {
47
+ label: answer.choice,
48
+ metadata: {
49
+ probabilities: answer.probabilities,
50
+ ...(confidence !== undefined ? { confidence } : {}),
51
+ modelId: result.response.modelId,
52
+ },
53
+ };
54
+ }
55
+ /**
56
+ * The decision model spec has no confidence field, but providers that
57
+ * report one (OpenAI, TypeSafe) put it in their provider metadata as
58
+ * `confidence[questionId]`.
59
+ */
60
+ function getConfidence(providerMetadata) {
61
+ if (typeof providerMetadata !== "object" || providerMetadata === null) {
62
+ return undefined;
63
+ }
64
+ for (const metadata of Object.values(providerMetadata)) {
65
+ const confidence = metadata
66
+ ?.confidence?.[QUESTION_ID];
67
+ if (typeof confidence === "number")
68
+ return confidence;
69
+ }
70
+ return undefined;
71
+ }
72
+ //# sourceMappingURL=decideClassification.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"decideClassification.js","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,IAAI,CAAC;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B;;;GAGG;AACH,MAAM,YAAY,GAChB,qFAAqF,CAAC;AAExF;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAK1C;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;QACvC,KAAK;QACL,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC;QACpC,SAAS,EAAE;YACT,CAAC,WAAW,CAAC,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,YAAY,EAAE,YAAY;gBAC1B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;QACD,SAAS,EAAE,mBAAmB,CAAC;YAC7B,UAAU,EAAE,sBAAsB;YAClC,SAAS;SACV,CAAC;KACH,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,gBAAyB;IAC9C,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,UAAU,GAAI,QAAqD;YACvE,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,OAAO,UAAU,KAAK,QAAQ;YAAE,OAAO,UAAU,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
@@ -0,0 +1,8 @@
1
+ import type { WithPrompt } from "../types/prompts.js";
2
+ /**
3
+ * Join a rendered AI SDK prompt into the plain-text state a decision model
4
+ * judges. Decision models have no roles, so system text, instructions and
5
+ * messages are joined in order.
6
+ */
7
+ export declare function promptToDecisionState(prompt: WithPrompt): string;
8
+ //# sourceMappingURL=decisionPrompt.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"decisionPrompt.d.ts","sourceRoot":"","sources":["../../../src/decision/decisionPrompt.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAahE"}
@@ -0,0 +1,66 @@
1
+ /**
2
+ * Join a rendered AI SDK prompt into the plain-text state a decision model
3
+ * judges. Decision models have no roles, so system text, instructions and
4
+ * messages are joined in order.
5
+ */
6
+ export function promptToDecisionState(prompt) {
7
+ const parts = [];
8
+ const { instructions, system, messages, prompt: promptContent } = prompt;
9
+ for (const value of [instructions, system]) {
10
+ if (value !== undefined)
11
+ parts.push(instructionsToText(value));
12
+ }
13
+ if (typeof promptContent === "string") {
14
+ parts.push(promptContent);
15
+ }
16
+ else if (Array.isArray(promptContent)) {
17
+ parts.push(messagesToText(promptContent));
18
+ }
19
+ if (messages !== undefined)
20
+ parts.push(messagesToText(messages));
21
+ return parts.filter((part) => part.length > 0).join("\n\n");
22
+ }
23
+ function messagesToText(messages) {
24
+ return messages
25
+ .map((message) => contentToText(message.content))
26
+ .filter((text) => text.length > 0)
27
+ .join("\n\n");
28
+ }
29
+ function instructionsToText(value) {
30
+ if (typeof value === "string")
31
+ return value;
32
+ if (Array.isArray(value)) {
33
+ return value
34
+ .map((item) => typeof item === "object" && item !== null && "content" in item
35
+ ? contentToText(item.content)
36
+ : "")
37
+ .filter((text) => text.length > 0)
38
+ .join("\n\n");
39
+ }
40
+ if (typeof value === "object" && value !== null && "content" in value) {
41
+ return contentToText(value.content);
42
+ }
43
+ return "";
44
+ }
45
+ /**
46
+ * Keep only the text of a message. Decision models judge text here; image
47
+ * and file parts are not forwarded.
48
+ */
49
+ function contentToText(content) {
50
+ if (typeof content === "string")
51
+ return content;
52
+ if (!Array.isArray(content))
53
+ return "";
54
+ return content
55
+ .map((part) => typeof part === "object" &&
56
+ part !== null &&
57
+ "type" in part &&
58
+ part.type === "text" &&
59
+ "text" in part &&
60
+ typeof part.text === "string"
61
+ ? part.text
62
+ : "")
63
+ .filter((text) => text.length > 0)
64
+ .join("\n");
65
+ }
66
+ //# sourceMappingURL=decisionPrompt.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"decisionPrompt.js","sourceRoot":"","sources":["../../../src/decision/decisionPrompt.ts"],"names":[],"mappings":"AAIA;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAkB;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,CAAC;IACzE,KAAK,MAAM,KAAK,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC;QAC3C,IAAI,KAAK,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC5B,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IACjE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,cAAc,CAAC,QAAiC;IACvD,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SAChD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK;aACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACZ,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,SAAS,IAAI,IAAI;YAC5D,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;YAC7B,CAAC,CAAC,EAAE,CACP;aACA,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;QACtE,OAAO,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,OAAgB;IACrC,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IAChD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACZ,OAAO,IAAI,KAAK,QAAQ;QACxB,IAAI,KAAK,IAAI;QACb,MAAM,IAAI,IAAI;QACd,IAAI,CAAC,IAAI,KAAK,MAAM;QACpB,MAAM,IAAI,IAAI;QACd,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;QAC3B,CAAC,CAAC,IAAI,CAAC,IAAI;QACX,CAAC,CAAC,EAAE,CACP;SACA,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}
@@ -1,7 +1,7 @@
1
1
  import type { LanguageModel } from "ai";
2
2
  import type { ClassificationChoicesMap, CreateClassificationEvaluatorArgs, EvaluatorFn, PromptTemplate, WithPromptTemplate } from "../types/index.js";
3
3
  import type { ObjectMapping } from "../types/data.js";
4
- import type { EvaluationModel } from "../utils/isEvaluationModel.js";
4
+ import type { DecisionModel } from "../utils/isDecisionModel.js";
5
5
  import { LLMEvaluator } from "./LLMEvaluator.js";
6
6
  /**
7
7
  * An LLM evaluator that performs evaluation via classification
@@ -16,7 +16,7 @@ export declare class ClassificationEvaluator<RecordType extends Record<string, u
16
16
  /**
17
17
  * The model to use for classification
18
18
  */
19
- readonly model: LanguageModel | EvaluationModel;
19
+ readonly model: LanguageModel | DecisionModel;
20
20
  /**
21
21
  * The choices to classify the example into
22
22
  */
@@ -1 +1 @@
1
- {"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EACV,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,eAAe,CAAC;IAChD;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;IAE3C,YAAY,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,EAQ9D;IACD,QAAQ,YAAa,UAAU,4CAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAKrC;CACF"}
1
+ {"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EACV,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAG9D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,aAAa,CAAC;IAC9C;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;IAE3C,YAAY,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,EAQ9D;IACD,QAAQ,YAAa,UAAU,4CAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAKrC;CACF"}
@@ -1 +1 @@
1
- {"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAUnD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,MAAM,OAAO,uBACX,SAAQ,YAAwB;IAGvB,WAAW,CAA0B;IACrC,cAAc,CAAiB;IACxC;;OAEG;IACK,wBAAwB,CAAuB;IACvD;;OAEG;IACM,KAAK,CAAkC;IAChD;;OAEG;IACM,OAAO,CAA2B;IAE3C,YAAY,IAAmD;QAC7D,KAAK,CAAC,IAAI,CAAC,CAAC;QACZ,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,kBAAkB,CAAa;YAChD,GAAG,IAAI;SACR,CAAC,CAAC;IACL,CAAC;IACD,QAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;QACjC,OAAO,IAAI,CAAC,WAAW,CACrB,IAAI,CAAC,YAAY;YACf,CAAC,CAAC,WAAW,CAAa,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;YACrD,CAAC,CAAC,OAAO,CACZ,CAAC;IACJ,CAAC,CAAC;IACF;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,oBAAoB,CAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,uBAAuB,CAAC;YACjC,GAAG,IAAI;YACP,YAAY;SACb,CAAC,CAAC;IACL,CAAC;CACF"}
1
+ {"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAUnD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,MAAM,OAAO,uBACX,SAAQ,YAAwB;IAGvB,WAAW,CAA0B;IACrC,cAAc,CAAiB;IACxC;;OAEG;IACK,wBAAwB,CAAuB;IACvD;;OAEG;IACM,KAAK,CAAgC;IAC9C;;OAEG;IACM,OAAO,CAA2B;IAE3C,YAAY,IAAmD;QAC7D,KAAK,CAAC,IAAI,CAAC,CAAC;QACZ,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,kBAAkB,CAAa;YAChD,GAAG,IAAI;SACR,CAAC,CAAC;IACL,CAAC;IACD,QAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;QACjC,OAAO,IAAI,CAAC,WAAW,CACrB,IAAI,CAAC,YAAY;YACf,CAAC,CAAC,WAAW,CAAa,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;YACrD,CAAC,CAAC,OAAO,CACZ,CAAC;IACJ,CAAC,CAAC;IACF;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,oBAAoB,CAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,uBAAuB,CAAC;YACjC,GAAG,IAAI;YACP,YAAY;SACb,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -18,11 +18,11 @@ export type ClassifyArgs = WithLLM & WithTelemetry & WithPrompt & {
18
18
  /**
19
19
  * A function that leverages an llm to perform a classification
20
20
  *
21
- * When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
22
- * classification is routed through `experimental_evaluate` as a single
23
- * choice question. Evaluation models cannot generate text, so no explanation
24
- * is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
25
- * are ignored on that path: `experimental_evaluate` does not emit spans yet.
21
+ * When given an AI SDK decision model (e.g. `openai.decisionModel("gpt-6-luna")`)
22
+ * the classification is routed through `experimental_decide` as a single
23
+ * choice question, with the prompt sent as plain text. Decision models cannot
24
+ * generate text, so no explanation is returned in that case, and
25
+ * `schemaName` / `schemaDescription` are ignored.
26
26
  */
27
27
  export declare function generateClassification(args: ClassifyArgs): Promise<ClassificationResult>;
28
28
  //# sourceMappingURL=generateClassification.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAMnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAgC/B"}
1
+ {"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAGnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAkC/B"}
@@ -1,26 +1,27 @@
1
- import { experimental_evaluate, generateObject } from "ai";
1
+ import { generateObject } from "ai";
2
2
  import { z } from "zod";
3
- import { getTelemetryIntegrations, tracer } from "../telemetry/index.js";
4
- import { isEvaluationModel, } from "../utils/isEvaluationModel.js";
3
+ import { decideClassification } from "../decision/decideClassification.js";
4
+ import { getTelemetryOptions } from "../telemetry/index.js";
5
+ import { isDecisionModel } from "../utils/isDecisionModel.js";
5
6
  /**
6
7
  * A function that leverages an llm to perform a classification
7
8
  *
8
- * When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
9
- * classification is routed through `experimental_evaluate` as a single
10
- * choice question. Evaluation models cannot generate text, so no explanation
11
- * is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
12
- * are ignored on that path: `experimental_evaluate` does not emit spans yet.
9
+ * When given an AI SDK decision model (e.g. `openai.decisionModel("gpt-6-luna")`)
10
+ * the classification is routed through `experimental_decide` as a single
11
+ * choice question, with the prompt sent as plain text. Decision models cannot
12
+ * generate text, so no explanation is returned in that case, and
13
+ * `schemaName` / `schemaDescription` are ignored.
13
14
  */
14
15
  export async function generateClassification(args) {
15
16
  const { labels, model, schemaName, schemaDescription, telemetry, ...prompt } = args;
16
- if (isEvaluationModel(model)) {
17
- return evaluateClassification({ model, labels, prompt });
17
+ if (isDecisionModel(model)) {
18
+ return decideClassification({
19
+ model,
20
+ labels,
21
+ prompt,
22
+ telemetry,
23
+ });
18
24
  }
19
- const telemetryOptions = {
20
- isEnabled: telemetry?.isEnabled ?? true,
21
- functionId: "generateClassification",
22
- integrations: getTelemetryIntegrations(telemetry?.tracer ?? tracer),
23
- };
24
25
  const result = await generateObject({
25
26
  model,
26
27
  schemaName,
@@ -29,7 +30,10 @@ export async function generateClassification(args) {
29
30
  explanation: z.string(), // We place the explanation in hopes it uses reasoning to explain the label.
30
31
  label: z.enum(labels),
31
32
  }),
32
- telemetry: telemetryOptions,
33
+ telemetry: getTelemetryOptions({
34
+ functionId: "generateClassification",
35
+ telemetry,
36
+ }),
33
37
  // AI SDK 7 rejects system messages inside `messages` by default; keep
34
38
  // accepting them since prompt templates may include system messages.
35
39
  allowSystemInMessages: true,
@@ -40,59 +44,4 @@ export async function generateClassification(args) {
40
44
  explanation: result.object.explanation,
41
45
  };
42
46
  }
43
- /**
44
- * Classify using an AI SDK evaluation model. The rendered prompt becomes the
45
- * shared evaluation state and the labels become the options of one choice
46
- * question. The model's label probabilities and resolved model id are
47
- * returned as metadata.
48
- */
49
- async function evaluateClassification(args) {
50
- const { model, labels, prompt } = args;
51
- const result = await experimental_evaluate({
52
- model,
53
- state: toEvaluationState(prompt),
54
- questions: {
55
- label: {
56
- type: "choice",
57
- instructions: "Read the prompt in the state and answer it by selecting exactly one of the choices.",
58
- criteria: Object.fromEntries(labels.map((label) => [label, null])),
59
- },
60
- },
61
- });
62
- const answer = result.answers.label;
63
- return {
64
- label: answer.choice,
65
- metadata: {
66
- probabilities: answer.probabilities,
67
- modelId: result.response.modelId,
68
- },
69
- };
70
- }
71
- /**
72
- * Convert an AI SDK prompt into JSON-compatible evaluation state.
73
- * A bare text prompt is passed through as a string; anything else
74
- * (instructions, system messages, message arrays) is serialized to JSON.
75
- */
76
- function toEvaluationState(prompt) {
77
- const { instructions, system, messages, prompt: promptContent } = prompt;
78
- if (typeof promptContent === "string" &&
79
- instructions === undefined &&
80
- system === undefined &&
81
- messages === undefined) {
82
- return promptContent;
83
- }
84
- const state = {};
85
- if (instructions !== undefined)
86
- state.instructions = instructions;
87
- if (system !== undefined)
88
- state.system = system;
89
- if (promptContent !== undefined)
90
- state.prompt = promptContent;
91
- if (messages !== undefined)
92
- state.messages = messages;
93
- // Round-trip through JSON so the state only contains JSON values
94
- // (strips `undefined`, turns Date/URL into strings). Binary file parts
95
- // are not supported by evaluation models and are not stripped here.
96
- return JSON.parse(JSON.stringify(state));
97
- }
98
47
  //# sourceMappingURL=generateClassification.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AAC3D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,wBAAwB,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAIhE,OAAO,EAEL,iBAAiB,GAClB,MAAM,4BAA4B,CAAC;AAkBpC;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAC1E,IAAI,CAAC;IAEP,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,sBAAsB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,gBAAgB,GAAG;QACvB,SAAS,EAAE,SAAS,EAAE,SAAS,IAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,YAAY,EAAE,wBAAwB,CAAC,SAAS,EAAE,MAAM,IAAI,MAAM,CAAC;KACpE,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,UAAU;QACV,iBAAiB;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC;QACF,SAAS,EAAE,gBAAgB;QAC3B,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI;QAC3B,GAAG,MAAM;KACV,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC;AAID;;;;;GAKG;AACH,KAAK,UAAU,sBAAsB,CAAC,IAIrC;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC;QACzC,KAAK;QACL,KAAK,EAAE,iBAAiB,CAAC,MAAM,CAAC;QAChC,SAAS,EAAE;YACT,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,YAAY,EACV,qFAAqF;gBACvF,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;KACF,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;IACpC,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,MAAkB;IAC3C,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,CAAC;IACzE,IACE,OAAO,aAAa,KAAK,QAAQ;QACjC,YAAY,KAAK,SAAS;QAC1B,MAAM,KAAK,SAAS;QACpB,QAAQ,KAAK,SAAS,EACtB,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,IAAI,YAAY,KAAK,SAAS;QAAE,KAAK,CAAC,YAAY,GAAG,YAAY,CAAC;IAClE,IAAI,MAAM,KAAK,SAAS;QAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IAChD,IAAI,aAAa,KAAK,SAAS;QAAE,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC;IAC9D,IAAI,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACtD,iEAAiE;IACjE,uEAAuE;IACvE,oEAAoE;IACpE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAoB,CAAC;AAC9D,CAAC"}
1
+ {"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAInD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAkB3D;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAC1E,IAAI,CAAC;IAEP,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,oBAAoB,CAAC;YAC1B,KAAK;YACL,MAAM;YACN,MAAM;YACN,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,UAAU;QACV,iBAAiB;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC;QACF,SAAS,EAAE,mBAAmB,CAAC;YAC7B,UAAU,EAAE,wBAAwB;YACpC,SAAS;SACV,CAAC;QACF,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI;QAC3B,GAAG,MAAM;KACV,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}
@@ -1,5 +1,6 @@
1
1
  import { type Tracer } from "@opentelemetry/api";
2
2
  import type { Telemetry } from "ai";
3
+ import type { TelemetryConfig } from "../types/otel.js";
3
4
  /**
4
5
  * Returns a lazy tracer that resolves from `trace.getTracer()` on every call,
5
6
  * so evaluator spans follow whichever provider is currently mounted as global.
@@ -18,4 +19,17 @@ export declare const tracer: Tracer;
18
19
  * used as-is so the call is not traced twice.
19
20
  */
20
21
  export declare function getTelemetryIntegrations(integrationTracer: Tracer): Telemetry[];
22
+ /**
23
+ * Builds the AI SDK `telemetry` call option for an evaluator call. Telemetry
24
+ * is on unless explicitly disabled, and spans go to the configured tracer or
25
+ * the default Phoenix tracer.
26
+ */
27
+ export declare function getTelemetryOptions(args: {
28
+ functionId: string;
29
+ telemetry?: TelemetryConfig;
30
+ }): {
31
+ isEnabled: boolean;
32
+ functionId: string;
33
+ integrations: Telemetry[];
34
+ };
21
35
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAIpC;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,IAAI,GAAE,MAA4B,GAAG,MAAM,CAUpE;AAED,eAAO,MAAM,MAAM,QAAc,CAAC;AAElC;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,iBAAiB,EAAE,MAAM,GACxB,SAAS,EAAE,CAsBb"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAIrD;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,IAAI,GAAE,MAA4B,GAAG,MAAM,CAUpE;AAED,eAAO,MAAM,MAAM,QAAc,CAAC;AAElC;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,iBAAiB,EAAE,MAAM,GACxB,SAAS,EAAE,CAwBb;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;;;;EAOA"}
@@ -46,7 +46,22 @@ export function getTelemetryIntegrations(integrationTracer) {
46
46
  usage: true,
47
47
  providerMetadata: true,
48
48
  schema: true,
49
+ // Record the state, questions and answers of decision model calls.
50
+ experimental_decision: true,
49
51
  }),
50
52
  ];
51
53
  }
54
+ /**
55
+ * Builds the AI SDK `telemetry` call option for an evaluator call. Telemetry
56
+ * is on unless explicitly disabled, and spans go to the configured tracer or
57
+ * the default Phoenix tracer.
58
+ */
59
+ export function getTelemetryOptions(args) {
60
+ const { functionId, telemetry } = args;
61
+ return {
62
+ isEnabled: telemetry?.isEnabled ?? true,
63
+ functionId,
64
+ integrations: getTelemetryIntegrations(telemetry?.tracer ?? tracer),
65
+ };
66
+ }
52
67
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAe,MAAM,oBAAoB,CAAC;AAGxD,MAAM,mBAAmB,GAAG,eAAe,CAAC;AAE5C;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,IAAI,GAAW,mBAAmB;IAC1D,OAAO;QACL,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO;YAClC,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACrE,CAAC;QACD,eAAe,CAAC,GAAG,IAAe;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrC,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,CAAC;KACQ,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CACtC,iBAAyB;IAEzB,MAAM,kBAAkB,GAAG,UAAU,CAAC,6BAA6B,IAAI,EAAE,CAAC;IAC1E,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,IAAI;IAClD,qEAAqE;IACrE,wBAAwB;IACxB,CAAC,WAAW,EAAE,EAAE,CACb,WAAoC,CAAC,MAAM,KAAK,iBAAiB,CACrE,CAAC;IACF,IAAI,oBAAoB,EAAE,CAAC;QACzB,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IACD,OAAO;QACL,GAAG,kBAAkB;QACrB,IAAI,aAAa,CAAC;YAChB,MAAM,EAAE,iBAAiB;YACzB,sEAAsE;YACtE,oCAAoC;YACpC,KAAK,EAAE,IAAI;YACX,gBAAgB,EAAE,IAAI;YACtB,MAAM,EAAE,IAAI;SACb,CAAC;KACH,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAe,MAAM,oBAAoB,CAAC;AAKxD,MAAM,mBAAmB,GAAG,eAAe,CAAC;AAE5C;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,IAAI,GAAW,mBAAmB;IAC1D,OAAO;QACL,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO;YAClC,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACrE,CAAC;QACD,eAAe,CAAC,GAAG,IAAe;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrC,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,CAAC;KACQ,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CACtC,iBAAyB;IAEzB,MAAM,kBAAkB,GAAG,UAAU,CAAC,6BAA6B,IAAI,EAAE,CAAC;IAC1E,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,IAAI;IAClD,qEAAqE;IACrE,wBAAwB;IACxB,CAAC,WAAW,EAAE,EAAE,CACb,WAAoC,CAAC,MAAM,KAAK,iBAAiB,CACrE,CAAC;IACF,IAAI,oBAAoB,EAAE,CAAC;QACzB,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IACD,OAAO;QACL,GAAG,kBAAkB;QACrB,IAAI,aAAa,CAAC;YAChB,MAAM,EAAE,iBAAiB;YACzB,sEAAsE;YACtE,oCAAoC;YACpC,KAAK,EAAE,IAAI;YACX,gBAAgB,EAAE,IAAI;YACtB,MAAM,EAAE,IAAI;YACZ,mEAAmE;YACnE,qBAAqB,EAAE,IAAI;SAC5B,CAAC;KACH,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAGnC;IACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IACvC,OAAO;QACL,SAAS,EAAE,SAAS,EAAE,SAAS,IAAI,IAAI;QACvC,UAAU;QACV,YAAY,EAAE,wBAAwB,CAAC,SAAS,EAAE,MAAM,IAAI,MAAM,CAAC;KACpE,CAAC;AACJ,CAAC"}