@arizeai/phoenix-evals 2.5.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/dist/esm/decision/decideClassification.d.ts +20 -0
  3. package/dist/esm/decision/decideClassification.d.ts.map +1 -0
  4. package/dist/esm/decision/decideClassification.js +72 -0
  5. package/dist/esm/decision/decideClassification.js.map +1 -0
  6. package/dist/esm/decision/decisionPrompt.d.ts +8 -0
  7. package/dist/esm/decision/decisionPrompt.d.ts.map +1 -0
  8. package/dist/esm/decision/decisionPrompt.js +66 -0
  9. package/dist/esm/decision/decisionPrompt.js.map +1 -0
  10. package/dist/esm/llm/ClassificationEvaluator.d.ts +2 -1
  11. package/dist/esm/llm/ClassificationEvaluator.d.ts.map +1 -1
  12. package/dist/esm/llm/ClassificationEvaluator.js.map +1 -1
  13. package/dist/esm/llm/generateClassification.d.ts +6 -0
  14. package/dist/esm/llm/generateClassification.d.ts.map +1 -1
  15. package/dist/esm/llm/generateClassification.js +21 -7
  16. package/dist/esm/llm/generateClassification.js.map +1 -1
  17. package/dist/esm/telemetry/index.d.ts +14 -0
  18. package/dist/esm/telemetry/index.d.ts.map +1 -1
  19. package/dist/esm/telemetry/index.js +15 -0
  20. package/dist/esm/telemetry/index.js.map +1 -1
  21. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  22. package/dist/esm/types/evals.d.ts +16 -2
  23. package/dist/esm/types/evals.d.ts.map +1 -1
  24. package/dist/esm/utils/index.d.ts +1 -0
  25. package/dist/esm/utils/index.d.ts.map +1 -1
  26. package/dist/esm/utils/index.js +1 -0
  27. package/dist/esm/utils/index.js.map +1 -1
  28. package/dist/esm/utils/isDecisionModel.d.ts +18 -0
  29. package/dist/esm/utils/isDecisionModel.d.ts.map +1 -0
  30. package/dist/esm/utils/isDecisionModel.js +13 -0
  31. package/dist/esm/utils/isDecisionModel.js.map +1 -0
  32. package/dist/src/decision/decideClassification.d.ts +20 -0
  33. package/dist/src/decision/decideClassification.d.ts.map +1 -0
  34. package/dist/src/decision/decideClassification.js +71 -0
  35. package/dist/src/decision/decideClassification.js.map +1 -0
  36. package/dist/src/decision/decisionPrompt.d.ts +8 -0
  37. package/dist/src/decision/decisionPrompt.d.ts.map +1 -0
  38. package/dist/src/decision/decisionPrompt.js +69 -0
  39. package/dist/src/decision/decisionPrompt.js.map +1 -0
  40. package/dist/src/llm/ClassificationEvaluator.d.ts +2 -1
  41. package/dist/src/llm/ClassificationEvaluator.d.ts.map +1 -1
  42. package/dist/src/llm/ClassificationEvaluator.js.map +1 -1
  43. package/dist/src/llm/generateClassification.d.ts +6 -0
  44. package/dist/src/llm/generateClassification.d.ts.map +1 -1
  45. package/dist/src/llm/generateClassification.js +20 -7
  46. package/dist/src/llm/generateClassification.js.map +1 -1
  47. package/dist/src/telemetry/index.d.ts +14 -0
  48. package/dist/src/telemetry/index.d.ts.map +1 -1
  49. package/dist/src/telemetry/index.js +17 -0
  50. package/dist/src/telemetry/index.js.map +1 -1
  51. package/dist/src/types/evals.d.ts +16 -2
  52. package/dist/src/types/evals.d.ts.map +1 -1
  53. package/dist/src/utils/index.d.ts +1 -0
  54. package/dist/src/utils/index.d.ts.map +1 -1
  55. package/dist/src/utils/index.js +1 -0
  56. package/dist/src/utils/index.js.map +1 -1
  57. package/dist/src/utils/isDecisionModel.d.ts +18 -0
  58. package/dist/src/utils/isDecisionModel.d.ts.map +1 -0
  59. package/dist/src/utils/isDecisionModel.js +16 -0
  60. package/dist/src/utils/isDecisionModel.js.map +1 -0
  61. package/dist/tsconfig.tsbuildinfo +1 -1
  62. package/docs/classification.mdx +43 -1
  63. package/package.json +13 -12
  64. package/src/decision/decideClassification.ts +85 -0
  65. package/src/decision/decisionPrompt.ts +70 -0
  66. package/src/llm/ClassificationEvaluator.ts +2 -1
  67. package/src/llm/generateClassification.ts +22 -7
  68. package/src/telemetry/index.ts +21 -0
  69. package/src/types/evals.ts +17 -3
  70. package/src/utils/index.ts +1 -0
  71. package/src/utils/isDecisionModel.ts +29 -0
@@ -1,4 +1,5 @@
1
1
  import type { LanguageModel } from "ai";
2
+ import type { DecisionModel } from "../utils/isDecisionModel.js";
2
3
  import type { ObjectMapping } from "./data.js";
3
4
  import type { WithTelemetry } from "./otel.js";
4
5
  import type { PromptTemplate } from "./templating.js";
@@ -12,7 +13,14 @@ export interface ExampleRecord<OutputType, InputType> {
12
13
  [key: string]: unknown;
13
14
  }
14
15
  export interface WithLLM {
15
- model: LanguageModel;
16
+ /**
17
+ * Either a regular AI SDK {@link LanguageModel}, or an AI SDK
18
+ * {@link DecisionModel} (e.g. `openai.decisionModel("gpt-6-luna")`) which
19
+ * classifies without generating text and therefore cannot produce an
20
+ * explanation. With a decision model, label probabilities are returned in
21
+ * the result's `metadata.probabilities`.
22
+ */
23
+ model: LanguageModel | DecisionModel;
16
24
  }
17
25
  export interface LLMEvaluationArgs extends WithLLM {
18
26
  }
@@ -35,6 +43,11 @@ export interface EvaluationResult {
35
43
  * @example "The model correctly identified the sentiment of the text."
36
44
  */
37
45
  explanation?: string;
46
+ /**
47
+ * Additional information about how the result was produced, e.g. the
48
+ * label probabilities reported by a decision model.
49
+ */
50
+ metadata?: Record<string, unknown>;
38
51
  }
39
52
  /**
40
53
  * The result of a classification
@@ -42,6 +55,7 @@ export interface EvaluationResult {
42
55
  export interface ClassificationResult {
43
56
  label: string;
44
57
  explanation?: string;
58
+ metadata?: Record<string, unknown>;
45
59
  }
46
60
  /**
47
61
  * The choice (e.g. the label and score mapping) of a classification based evaluation
@@ -58,7 +72,7 @@ export type ClassificationChoicesMap = Record<string, number>;
58
72
  * The arguments for creating a classification-based evaluator
59
73
  */
60
74
  export interface CreateClassifierArgs extends WithTelemetry {
61
- model: LanguageModel;
75
+ model: LanguageModel | DecisionModel;
62
76
  /**
63
77
  * The choices to classify the example into.
64
78
  * e.g. { "correct": 1, "incorrect": 0 }
@@ -1 +1 @@
1
- {"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAExE;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
1
+ {"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB;;;;;;OAMG;IACH,KAAK,EAAE,aAAa,GAAG,aAAa,CAAC;CACtC;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,GAAG,aAAa,CAAC;IACrC;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAExE;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
@@ -1,2 +1,3 @@
1
1
  export * from "./bindEvaluator.js";
2
+ export * from "./isDecisionModel.js";
2
3
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC"}
@@ -1,2 +1,3 @@
1
1
  export * from "./bindEvaluator.js";
2
+ export * from "./isDecisionModel.js";
2
3
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC"}
@@ -0,0 +1,18 @@
1
+ import type { Experimental_DecisionModel, LanguageModel } from "ai";
2
+ /**
3
+ * An AI SDK decision model instance, e.g. `openai.decisionModel("gpt-6-luna")`
4
+ * or TypeSafe's Jev. Unlike a {@link LanguageModel}, a decision model only
5
+ * answers typed questions (here, which label applies) and cannot generate
6
+ * free-form text, so classifications made with it have no explanation.
7
+ *
8
+ * Gateway string IDs are excluded: a plain string is treated as a language
9
+ * model. Wrap the ID with the provider's `decisionModel()` instead.
10
+ */
11
+ export type DecisionModel = Exclude<Experimental_DecisionModel, string>;
12
+ /**
13
+ * Type guard that distinguishes an AI SDK decision model from a language model.
14
+ * Decision models expose `doDecide` (or `doEvaluate` for providers written
15
+ * against the older evaluation model spec) instead of `doGenerate`.
16
+ */
17
+ export declare function isDecisionModel(model: LanguageModel | DecisionModel): model is DecisionModel;
18
+ //# sourceMappingURL=isDecisionModel.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"isDecisionModel.d.ts","sourceRoot":"","sources":["../../../src/utils/isDecisionModel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEpE;;;;;;;;GAQG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,CAAC,0BAA0B,EAAE,MAAM,CAAC,CAAC;AAExE;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,aAAa,GAAG,aAAa,GACnC,KAAK,IAAI,aAAa,CAQxB"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Type guard that distinguishes an AI SDK decision model from a language model.
3
+ * Decision models expose `doDecide` (or `doEvaluate` for providers written
4
+ * against the older evaluation model spec) instead of `doGenerate`.
5
+ */
6
+ export function isDecisionModel(model) {
7
+ if (typeof model !== "object" || model === null) {
8
+ return false;
9
+ }
10
+ return (("doDecide" in model && typeof model.doDecide === "function") ||
11
+ ("doEvaluate" in model && typeof model.doEvaluate === "function"));
12
+ }
13
+ //# sourceMappingURL=isDecisionModel.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"isDecisionModel.js","sourceRoot":"","sources":["../../../src/utils/isDecisionModel.ts"],"names":[],"mappings":"AAaA;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAoC;IAEpC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,CACL,CAAC,UAAU,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,UAAU,CAAC;QAC7D,CAAC,YAAY,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,UAAU,CAAC,CAClE,CAAC;AACJ,CAAC"}
@@ -0,0 +1,20 @@
1
+ import type { ClassificationResult } from "../types/evals";
2
+ import type { TelemetryConfig } from "../types/otel";
3
+ import type { WithPrompt } from "../types/prompts";
4
+ import type { DecisionModel } from "../utils/isDecisionModel";
5
+ /**
6
+ * Classify with an AI SDK decision model by asking one choice question whose
7
+ * options are the labels. Decision models don't generate text, so the
8
+ * result has no explanation. Label probabilities, the provider's confidence
9
+ * (when it reports one) and the resolved model ID go in the metadata.
10
+ *
11
+ * Choice limits and refusals are left to the AI SDK and the provider: a
12
+ * refusal throws the AI SDK's `Experimental_DecisionRefusalError`.
13
+ */
14
+ export declare function decideClassification(args: {
15
+ model: DecisionModel;
16
+ labels: readonly string[];
17
+ prompt: WithPrompt;
18
+ telemetry?: TelemetryConfig;
19
+ }): Promise<ClassificationResult>;
20
+ //# sourceMappingURL=decideClassification.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"decideClassification.d.ts","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAe9D;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA2BhC"}
@@ -0,0 +1,71 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.decideClassification = decideClassification;
4
+ /**
5
+ * The only place phoenix-evals calls the AI SDK's experimental decision API.
6
+ * `experimental_decide` and the decision model spec may change in AI SDK
7
+ * patch releases, so keep everything that depends on their shape in here.
8
+ */
9
+ const ai_1 = require("ai");
10
+ const telemetry_1 = require("../telemetry");
11
+ const decisionPrompt_1 = require("./decisionPrompt");
12
+ /**
13
+ * The ID of the single choice question asked per classification.
14
+ */
15
+ const QUESTION_ID = "label";
16
+ /**
17
+ * The question's instructions. The rendered prompt, rubric included, is sent
18
+ * as the state.
19
+ */
20
+ const INSTRUCTIONS = "Read the prompt in the state and answer it by selecting exactly one of the choices.";
21
+ /**
22
+ * Classify with an AI SDK decision model by asking one choice question whose
23
+ * options are the labels. Decision models don't generate text, so the
24
+ * result has no explanation. Label probabilities, the provider's confidence
25
+ * (when it reports one) and the resolved model ID go in the metadata.
26
+ *
27
+ * Choice limits and refusals are left to the AI SDK and the provider: a
28
+ * refusal throws the AI SDK's `Experimental_DecisionRefusalError`.
29
+ */
30
+ async function decideClassification(args) {
31
+ const { model, labels, prompt, telemetry } = args;
32
+ const result = await (0, ai_1.experimental_decide)({
33
+ model,
34
+ state: (0, decisionPrompt_1.promptToDecisionState)(prompt),
35
+ questions: {
36
+ [QUESTION_ID]: {
37
+ type: "choice",
38
+ instructions: INSTRUCTIONS,
39
+ criteria: Object.fromEntries(labels.map((label) => [label, null])),
40
+ },
41
+ },
42
+ telemetry: (0, telemetry_1.getTelemetryOptions)({
43
+ functionId: "decideClassification",
44
+ telemetry,
45
+ }),
46
+ });
47
+ const answer = result.answers[QUESTION_ID];
48
+ const confidence = getConfidence(result.providerMetadata);
49
+ return {
50
+ label: answer.choice,
51
+ metadata: Object.assign(Object.assign({ probabilities: answer.probabilities }, (confidence !== undefined ? { confidence } : {})), { modelId: result.response.modelId }),
52
+ };
53
+ }
54
+ /**
55
+ * The decision model spec has no confidence field, but providers that
56
+ * report one (OpenAI, TypeSafe) put it in their provider metadata as
57
+ * `confidence[questionId]`.
58
+ */
59
+ function getConfidence(providerMetadata) {
60
+ var _a;
61
+ if (typeof providerMetadata !== "object" || providerMetadata === null) {
62
+ return undefined;
63
+ }
64
+ for (const metadata of Object.values(providerMetadata)) {
65
+ const confidence = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.confidence) === null || _a === void 0 ? void 0 : _a[QUESTION_ID];
66
+ if (typeof confidence === "number")
67
+ return confidence;
68
+ }
69
+ return undefined;
70
+ }
71
+ //# sourceMappingURL=decideClassification.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"decideClassification.js","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":";;;AAAA;;;;GAIG;AACH,2BAAyC;AAEzC,4CAAmD;AAKnD,qDAAyD;AAEzD;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B;;;GAGG;AACH,MAAM,YAAY,GAChB,qFAAqF,CAAC;AAExF;;;;;;;;GAQG;AACI,KAAK,+BAA+B,IAK1C;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,IAAA,wBAAmB,EAAC;QACvC,KAAK;QACL,KAAK,EAAE,IAAA,sCAAqB,EAAC,MAAM,CAAC;QACpC,SAAS,EAAE;YACT,CAAC,WAAW,CAAC,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,YAAY,EAAE,YAAY;gBAC1B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;QACD,SAAS,EAAE,IAAA,+BAAmB,EAAC;YAC7B,UAAU,EAAE,sBAAsB;YAClC,SAAS;SACV,CAAC;KACH,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,gCACN,aAAa,EAAE,MAAM,CAAC,aAAa,IAChC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KACnD,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,GACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,gBAAyB;;IAC9C,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,UAAU,GAAG,MAAC,QAAqD,aAArD,QAAQ,uBAAR,QAAQ,CACxB,UAAU,0CAAG,WAAW,CAAC,CAAC;QAC9B,IAAI,OAAO,UAAU,KAAK,QAAQ;YAAE,OAAO,UAAU,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
@@ -0,0 +1,8 @@
1
+ import type { WithPrompt } from "../types/prompts";
2
+ /**
3
+ * Join a rendered AI SDK prompt into the plain-text state a decision model
4
+ * judges. Decision models have no roles, so system text, instructions and
5
+ * messages are joined in order.
6
+ */
7
+ export declare function promptToDecisionState(prompt: WithPrompt): string;
8
+ //# sourceMappingURL=decisionPrompt.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"decisionPrompt.d.ts","sourceRoot":"","sources":["../../../src/decision/decisionPrompt.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAahE"}
@@ -0,0 +1,69 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.promptToDecisionState = promptToDecisionState;
4
+ /**
5
+ * Join a rendered AI SDK prompt into the plain-text state a decision model
6
+ * judges. Decision models have no roles, so system text, instructions and
7
+ * messages are joined in order.
8
+ */
9
+ function promptToDecisionState(prompt) {
10
+ const parts = [];
11
+ const { instructions, system, messages, prompt: promptContent } = prompt;
12
+ for (const value of [instructions, system]) {
13
+ if (value !== undefined)
14
+ parts.push(instructionsToText(value));
15
+ }
16
+ if (typeof promptContent === "string") {
17
+ parts.push(promptContent);
18
+ }
19
+ else if (Array.isArray(promptContent)) {
20
+ parts.push(messagesToText(promptContent));
21
+ }
22
+ if (messages !== undefined)
23
+ parts.push(messagesToText(messages));
24
+ return parts.filter((part) => part.length > 0).join("\n\n");
25
+ }
26
+ function messagesToText(messages) {
27
+ return messages
28
+ .map((message) => contentToText(message.content))
29
+ .filter((text) => text.length > 0)
30
+ .join("\n\n");
31
+ }
32
+ function instructionsToText(value) {
33
+ if (typeof value === "string")
34
+ return value;
35
+ if (Array.isArray(value)) {
36
+ return value
37
+ .map((item) => typeof item === "object" && item !== null && "content" in item
38
+ ? contentToText(item.content)
39
+ : "")
40
+ .filter((text) => text.length > 0)
41
+ .join("\n\n");
42
+ }
43
+ if (typeof value === "object" && value !== null && "content" in value) {
44
+ return contentToText(value.content);
45
+ }
46
+ return "";
47
+ }
48
+ /**
49
+ * Keep only the text of a message. Decision models judge text here; image
50
+ * and file parts are not forwarded.
51
+ */
52
+ function contentToText(content) {
53
+ if (typeof content === "string")
54
+ return content;
55
+ if (!Array.isArray(content))
56
+ return "";
57
+ return content
58
+ .map((part) => typeof part === "object" &&
59
+ part !== null &&
60
+ "type" in part &&
61
+ part.type === "text" &&
62
+ "text" in part &&
63
+ typeof part.text === "string"
64
+ ? part.text
65
+ : "")
66
+ .filter((text) => text.length > 0)
67
+ .join("\n");
68
+ }
69
+ //# sourceMappingURL=decisionPrompt.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"decisionPrompt.js","sourceRoot":"","sources":["../../../src/decision/decisionPrompt.ts"],"names":[],"mappings":";;;AAIA;;;;GAIG;AACH,+BAAsC,MAAkB;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,CAAC;IACzE,KAAK,MAAM,KAAK,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC;QAC3C,IAAI,KAAK,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC5B,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IACjE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,cAAc,CAAC,QAAiC;IACvD,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SAChD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK;aACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACZ,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,SAAS,IAAI,IAAI;YAC5D,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;YAC7B,CAAC,CAAC,EAAE,CACP;aACA,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;QACtE,OAAO,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,OAAgB;IACrC,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IAChD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACZ,OAAO,IAAI,KAAK,QAAQ;QACxB,IAAI,KAAK,IAAI;QACb,MAAM,IAAI,IAAI;QACd,IAAI,CAAC,IAAI,KAAK,MAAM;QACpB,MAAM,IAAI,IAAI;QACd,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;QAC3B,CAAC,CAAC,IAAI,CAAC,IAAI;QACX,CAAC,CAAC,EAAE,CACP;SACA,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}
@@ -1,6 +1,7 @@
1
1
  import type { LanguageModel } from "ai";
2
2
  import type { ClassificationChoicesMap, CreateClassificationEvaluatorArgs, EvaluatorFn, PromptTemplate, WithPromptTemplate } from "../types";
3
3
  import type { ObjectMapping } from "../types/data";
4
+ import type { DecisionModel } from "../utils/isDecisionModel";
4
5
  import { LLMEvaluator } from "./LLMEvaluator";
5
6
  /**
6
7
  * An LLM evaluator that performs evaluation via classification
@@ -15,7 +16,7 @@ export declare class ClassificationEvaluator<RecordType extends Record<string, u
15
16
  /**
16
17
  * The model to use for classification
17
18
  */
18
- readonly model: LanguageModel;
19
+ readonly model: LanguageModel | DecisionModel;
19
20
  /**
20
21
  * The choices to classify the example into
21
22
  */
@@ -1 +1 @@
1
- {"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EACV,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;IAE3C,YAAY,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,EAQ9D;IACD,QAAQ,YAAa,UAAU,kDAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAKrC;CACF"}
1
+ {"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EACV,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAG9D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,aAAa,CAAC;IAC9C;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;IAE3C,YAAY,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,EAQ9D;IACD,QAAQ,YAAa,UAAU,kDAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAKrC;CACF"}
@@ -1 +1 @@
1
- {"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":";;;AAEA,0CAAmD;AASnD,oEAA0D;AAC1D,6DAA0D;AAC1D,iDAA8C;AAE9C;;GAEG;AACH,6BACE,SAAQ,2BAAwB;IAkBhC,YAAY,IAAmD;QAC7D,KAAK,CAAC,IAAI,CAAC,CAAC;QAQd,aAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;YACjC,OAAO,IAAI,CAAC,WAAW,CACrB,IAAI,CAAC,YAAY;gBACf,CAAC,CAAC,IAAA,gCAAW,EAAa,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;gBACrD,CAAC,CAAC,OAAO,CACZ,CAAC;QACJ,CAAC,CAAC;QAbA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAA,uCAAkB,oBAChC,IAAI,EACP,CAAC;IACL,CAAC;IAQD;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,IAAA,+BAAoB,EAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,uBAAuB,iCAC7B,IAAI,KACP,YAAY,IACZ,CAAC;IACL,CAAC;CACF"}
1
+ {"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":";;;AAEA,0CAAmD;AAUnD,oEAA0D;AAC1D,6DAA0D;AAC1D,iDAA8C;AAE9C;;GAEG;AACH,6BACE,SAAQ,2BAAwB;IAkBhC,YAAY,IAAmD;QAC7D,KAAK,CAAC,IAAI,CAAC,CAAC;QAQd,aAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;YACjC,OAAO,IAAI,CAAC,WAAW,CACrB,IAAI,CAAC,YAAY;gBACf,CAAC,CAAC,IAAA,gCAAW,EAAa,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;gBACrD,CAAC,CAAC,OAAO,CACZ,CAAC;QACJ,CAAC,CAAC;QAbA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAA,uCAAkB,oBAChC,IAAI,EACP,CAAC;IACL,CAAC;IAQD;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,IAAA,+BAAoB,EAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,uBAAuB,iCAC7B,IAAI,KACP,YAAY,IACZ,CAAC;IACL,CAAC;CACF"}
@@ -17,6 +17,12 @@ export type ClassifyArgs = WithLLM & WithTelemetry & WithPrompt & {
17
17
  };
18
18
  /**
19
19
  * A function that leverages an llm to perform a classification
20
+ *
21
+ * When given an AI SDK decision model (e.g. `openai.decisionModel("gpt-6-luna")`)
22
+ * the classification is routed through `experimental_decide` as a single
23
+ * choice question, with the prompt sent as plain text. Decision models cannot
24
+ * generate text, so no explanation is returned in that case, and
25
+ * `schemaName` / `schemaDescription` are ignored.
20
26
  */
21
27
  export declare function generateClassification(args: ClassifyArgs): Promise<ClassificationResult>;
22
28
  //# sourceMappingURL=generateClassification.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CA4B/B"}
1
+ {"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAGnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAkC/B"}
@@ -14,24 +14,37 @@ Object.defineProperty(exports, "__esModule", { value: true });
14
14
  exports.generateClassification = generateClassification;
15
15
  const ai_1 = require("ai");
16
16
  const zod_1 = require("zod");
17
+ const decideClassification_1 = require("../decision/decideClassification");
17
18
  const telemetry_1 = require("../telemetry");
19
+ const isDecisionModel_1 = require("../utils/isDecisionModel");
18
20
  /**
19
21
  * A function that leverages an llm to perform a classification
22
+ *
23
+ * When given an AI SDK decision model (e.g. `openai.decisionModel("gpt-6-luna")`)
24
+ * the classification is routed through `experimental_decide` as a single
25
+ * choice question, with the prompt sent as plain text. Decision models cannot
26
+ * generate text, so no explanation is returned in that case, and
27
+ * `schemaName` / `schemaDescription` are ignored.
20
28
  */
21
29
  async function generateClassification(args) {
22
- var _a, _b;
23
30
  const { labels, model, schemaName, schemaDescription, telemetry } = args, prompt = __rest(args, ["labels", "model", "schemaName", "schemaDescription", "telemetry"]);
24
- const telemetryOptions = {
25
- isEnabled: (_a = telemetry === null || telemetry === void 0 ? void 0 : telemetry.isEnabled) !== null && _a !== void 0 ? _a : true,
26
- functionId: "generateClassification",
27
- integrations: (0, telemetry_1.getTelemetryIntegrations)((_b = telemetry === null || telemetry === void 0 ? void 0 : telemetry.tracer) !== null && _b !== void 0 ? _b : telemetry_1.tracer),
28
- };
31
+ if ((0, isDecisionModel_1.isDecisionModel)(model)) {
32
+ return (0, decideClassification_1.decideClassification)({
33
+ model,
34
+ labels,
35
+ prompt,
36
+ telemetry,
37
+ });
38
+ }
29
39
  const result = await (0, ai_1.generateObject)(Object.assign({ model,
30
40
  schemaName,
31
41
  schemaDescription, schema: zod_1.z.object({
32
42
  explanation: zod_1.z.string(), // We place the explanation in hopes it uses reasoning to explain the label.
33
43
  label: zod_1.z.enum(labels),
34
- }), telemetry: telemetryOptions,
44
+ }), telemetry: (0, telemetry_1.getTelemetryOptions)({
45
+ functionId: "generateClassification",
46
+ telemetry,
47
+ }),
35
48
  // AI SDK 7 rejects system messages inside `messages` by default; keep
36
49
  // accepting them since prompt templates may include system messages.
37
50
  allowSystemInMessages: true }, prompt));
@@ -1 +1 @@
1
- {"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,2BAAoC;AACpC,6BAAwB;AAExB,4CAAgE;AAoBhE;;GAEG;AACI,KAAK,iCACV,IAAkB;;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,KAC7D,IAAI,EAD8D,MAAM,UACxE,IAAI,EADA,mEAAsE,CACtE,CAAC;IAEP,MAAM,gBAAgB,GAAG;QACvB,SAAS,QAAE,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,SAAS,mCAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,YAAY,EAAE,IAAA,oCAAwB,QAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,mCAAI,kBAAM,CAAC;KACpE,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAc,kBACjC,KAAK;QACL,UAAU;QACV,iBAAiB,EACjB,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC,EACF,SAAS,EAAE,gBAAgB;QAC3B,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI,IACxB,MAAM,EACT,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,2BAAoC;AACpC,6BAAwB;AAExB,2EAAwE;AACxE,4CAAmD;AAInD,8DAA2D;AAkB3D;;;;;;;;GAQG;AACI,KAAK,iCACV,IAAkB;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,KAC7D,IAAI,EAD8D,MAAM,UACxE,IAAI,EADA,mEAAsE,CACtE,CAAC;IAEP,IAAI,IAAA,iCAAe,EAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAA,2CAAoB,EAAC;YAC1B,KAAK;YACL,MAAM;YACN,MAAM;YACN,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAc,kBACjC,KAAK;QACL,UAAU;QACV,iBAAiB,EACjB,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC,EACF,SAAS,EAAE,IAAA,+BAAmB,EAAC;YAC7B,UAAU,EAAE,wBAAwB;YACpC,SAAS;SACV,CAAC;QACF,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI,IACxB,MAAM,EACT,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}
@@ -1,5 +1,6 @@
1
1
  import { type Tracer } from "@opentelemetry/api";
2
2
  import type { Telemetry } from "ai";
3
+ import type { TelemetryConfig } from "../types/otel";
3
4
  /**
4
5
  * Returns a lazy tracer that resolves from `trace.getTracer()` on every call,
5
6
  * so evaluator spans follow whichever provider is currently mounted as global.
@@ -18,4 +19,17 @@ export declare const tracer: Tracer;
18
19
  * used as-is so the call is not traced twice.
19
20
  */
20
21
  export declare function getTelemetryIntegrations(integrationTracer: Tracer): Telemetry[];
22
+ /**
23
+ * Builds the AI SDK `telemetry` call option for an evaluator call. Telemetry
24
+ * is on unless explicitly disabled, and spans go to the configured tracer or
25
+ * the default Phoenix tracer.
26
+ */
27
+ export declare function getTelemetryOptions(args: {
28
+ functionId: string;
29
+ telemetry?: TelemetryConfig;
30
+ }): {
31
+ isEnabled: boolean;
32
+ functionId: string;
33
+ integrations: Telemetry[];
34
+ };
21
35
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAIpC;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,IAAI,GAAE,MAA4B,GAAG,MAAM,CAUpE;AAED,eAAO,MAAM,MAAM,QAAc,CAAC;AAElC;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,iBAAiB,EAAE,MAAM,GACxB,SAAS,EAAE,CAsBb"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAIrD;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,IAAI,GAAE,MAA4B,GAAG,MAAM,CAUpE;AAED,eAAO,MAAM,MAAM,QAAc,CAAC;AAElC;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,iBAAiB,EAAE,MAAM,GACxB,SAAS,EAAE,CAwBb;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;;;;EAOA"}
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.tracer = void 0;
4
4
  exports.getTracer = getTracer;
5
5
  exports.getTelemetryIntegrations = getTelemetryIntegrations;
6
+ exports.getTelemetryOptions = getTelemetryOptions;
6
7
  const otel_1 = require("@ai-sdk/otel");
7
8
  const api_1 = require("@opentelemetry/api");
8
9
  const DEFAULT_TRACER_NAME = "phoenix-evals";
@@ -52,7 +53,23 @@ function getTelemetryIntegrations(integrationTracer) {
52
53
  usage: true,
53
54
  providerMetadata: true,
54
55
  schema: true,
56
+ // Record the state, questions and answers of decision model calls.
57
+ experimental_decision: true,
55
58
  }),
56
59
  ];
57
60
  }
61
+ /**
62
+ * Builds the AI SDK `telemetry` call option for an evaluator call. Telemetry
63
+ * is on unless explicitly disabled, and spans go to the configured tracer or
64
+ * the default Phoenix tracer.
65
+ */
66
+ function getTelemetryOptions(args) {
67
+ var _a, _b;
68
+ const { functionId, telemetry } = args;
69
+ return {
70
+ isEnabled: (_a = telemetry === null || telemetry === void 0 ? void 0 : telemetry.isEnabled) !== null && _a !== void 0 ? _a : true,
71
+ functionId,
72
+ integrations: getTelemetryIntegrations((_b = telemetry === null || telemetry === void 0 ? void 0 : telemetry.tracer) !== null && _b !== void 0 ? _b : exports.tracer),
73
+ };
74
+ }
58
75
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":";;;;;AAAA,uCAA6C;AAC7C,4CAAwD;AAGxD,MAAM,mBAAmB,GAAG,eAAe,CAAC;AAE5C;;;;;;GAMG;AACH,mBAA0B,IAAI,GAAW,mBAAmB;IAC1D,OAAO;QACL,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO;YAClC,OAAO,WAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACrE,CAAC;QACD,eAAe,CAAC,GAAG,IAAe;YAChC,MAAM,MAAM,GAAG,WAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrC,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,CAAC;KACQ,CAAC;AACd,CAAC;AAEY,QAAA,MAAM,GAAG,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,kCACE,iBAAyB;;IAEzB,MAAM,kBAAkB,SAAG,UAAU,CAAC,6BAA6B,mCAAI,EAAE,CAAC;IAC1E,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,IAAI;IAClD,qEAAqE;IACrE,wBAAwB;IACxB,CAAC,WAAW,EAAE,EAAE,CACb,WAAoC,CAAC,MAAM,KAAK,iBAAiB,CACrE,CAAC;IACF,IAAI,oBAAoB,EAAE,CAAC;QACzB,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IACD,OAAO;QACL,GAAG,kBAAkB;QACrB,IAAI,oBAAa,CAAC;YAChB,MAAM,EAAE,iBAAiB;YACzB,sEAAsE;YACtE,oCAAoC;YACpC,KAAK,EAAE,IAAI;YACX,gBAAgB,EAAE,IAAI;YACtB,MAAM,EAAE,IAAI;SACb,CAAC;KACH,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":";;;;;;AAAA,uCAA6C;AAC7C,4CAAwD;AAKxD,MAAM,mBAAmB,GAAG,eAAe,CAAC;AAE5C;;;;;;GAMG;AACH,mBAA0B,IAAI,GAAW,mBAAmB;IAC1D,OAAO;QACL,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO;YAClC,OAAO,WAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACrE,CAAC;QACD,eAAe,CAAC,GAAG,IAAe;YAChC,MAAM,MAAM,GAAG,WAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrC,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,CAAC;KACQ,CAAC;AACd,CAAC;AAEY,QAAA,MAAM,GAAG,SAAS,EAAE,CAAC;AAElC;;;;;;;GAOG;AACH,kCACE,iBAAyB;;IAEzB,MAAM,kBAAkB,SAAG,UAAU,CAAC,6BAA6B,mCAAI,EAAE,CAAC;IAC1E,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,IAAI;IAClD,qEAAqE;IACrE,wBAAwB;IACxB,CAAC,WAAW,EAAE,EAAE,CACb,WAAoC,CAAC,MAAM,KAAK,iBAAiB,CACrE,CAAC;IACF,IAAI,oBAAoB,EAAE,CAAC;QACzB,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IACD,OAAO;QACL,GAAG,kBAAkB;QACrB,IAAI,oBAAa,CAAC;YAChB,MAAM,EAAE,iBAAiB;YACzB,sEAAsE;YACtE,oCAAoC;YACpC,KAAK,EAAE,IAAI;YACX,gBAAgB,EAAE,IAAI;YACtB,MAAM,EAAE,IAAI;YACZ,mEAAmE;YACnE,qBAAqB,EAAE,IAAI;SAC5B,CAAC;KACH,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,6BAAoC,IAGnC;;IACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IACvC,OAAO;QACL,SAAS,QAAE,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,SAAS,mCAAI,IAAI;QACvC,UAAU;QACV,YAAY,EAAE,wBAAwB,OAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,mCAAI,QAAA,MAAM,CAAC;KACpE,CAAC;AACJ,CAAC"}
@@ -1,4 +1,5 @@
1
1
  import type { LanguageModel } from "ai";
2
+ import type { DecisionModel } from "../utils/isDecisionModel";
2
3
  import type { ObjectMapping } from "./data";
3
4
  import type { WithTelemetry } from "./otel";
4
5
  import type { PromptTemplate } from "./templating";
@@ -12,7 +13,14 @@ export interface ExampleRecord<OutputType, InputType> {
12
13
  [key: string]: unknown;
13
14
  }
14
15
  export interface WithLLM {
15
- model: LanguageModel;
16
+ /**
17
+ * Either a regular AI SDK {@link LanguageModel}, or an AI SDK
18
+ * {@link DecisionModel} (e.g. `openai.decisionModel("gpt-6-luna")`) which
19
+ * classifies without generating text and therefore cannot produce an
20
+ * explanation. With a decision model, label probabilities are returned in
21
+ * the result's `metadata.probabilities`.
22
+ */
23
+ model: LanguageModel | DecisionModel;
16
24
  }
17
25
  export interface LLMEvaluationArgs extends WithLLM {
18
26
  }
@@ -35,6 +43,11 @@ export interface EvaluationResult {
35
43
  * @example "The model correctly identified the sentiment of the text."
36
44
  */
37
45
  explanation?: string;
46
+ /**
47
+ * Additional information about how the result was produced, e.g. the
48
+ * label probabilities reported by a decision model.
49
+ */
50
+ metadata?: Record<string, unknown>;
38
51
  }
39
52
  /**
40
53
  * The result of a classification
@@ -42,6 +55,7 @@ export interface EvaluationResult {
42
55
  export interface ClassificationResult {
43
56
  label: string;
44
57
  explanation?: string;
58
+ metadata?: Record<string, unknown>;
45
59
  }
46
60
  /**
47
61
  * The choice (e.g. the label and score mapping) of a classification based evaluation
@@ -58,7 +72,7 @@ export type ClassificationChoicesMap = Record<string, number>;
58
72
  * The arguments for creating a classification-based evaluator
59
73
  */
60
74
  export interface CreateClassifierArgs extends WithTelemetry {
61
- model: LanguageModel;
75
+ model: LanguageModel | DecisionModel;
62
76
  /**
63
77
  * The choices to classify the example into.
64
78
  * e.g. { "correct": 1, "incorrect": 0 }
@@ -1 +1 @@
1
- {"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAExE;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
1
+ {"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB;;;;;;OAMG;IACH,KAAK,EAAE,aAAa,GAAG,aAAa,CAAC;CACtC;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,GAAG,aAAa,CAAC;IACrC;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAExE;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
@@ -1,2 +1,3 @@
1
1
  export * from "./bindEvaluator";
2
+ export * from "./isDecisionModel";
2
3
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC"}
@@ -15,4 +15,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
17
  __exportStar(require("./bindEvaluator"), exports);
18
+ __exportStar(require("./isDecisionModel"), exports);
18
19
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/utils/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC;AAChC,oDAAkC"}
@@ -0,0 +1,18 @@
1
+ import type { Experimental_DecisionModel, LanguageModel } from "ai";
2
+ /**
3
+ * An AI SDK decision model instance, e.g. `openai.decisionModel("gpt-6-luna")`
4
+ * or TypeSafe's Jev. Unlike a {@link LanguageModel}, a decision model only
5
+ * answers typed questions (here, which label applies) and cannot generate
6
+ * free-form text, so classifications made with it have no explanation.
7
+ *
8
+ * Gateway string IDs are excluded: a plain string is treated as a language
9
+ * model. Wrap the ID with the provider's `decisionModel()` instead.
10
+ */
11
+ export type DecisionModel = Exclude<Experimental_DecisionModel, string>;
12
+ /**
13
+ * Type guard that distinguishes an AI SDK decision model from a language model.
14
+ * Decision models expose `doDecide` (or `doEvaluate` for providers written
15
+ * against the older evaluation model spec) instead of `doGenerate`.
16
+ */
17
+ export declare function isDecisionModel(model: LanguageModel | DecisionModel): model is DecisionModel;
18
+ //# sourceMappingURL=isDecisionModel.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"isDecisionModel.d.ts","sourceRoot":"","sources":["../../../src/utils/isDecisionModel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEpE;;;;;;;;GAQG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,CAAC,0BAA0B,EAAE,MAAM,CAAC,CAAC;AAExE;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,aAAa,GAAG,aAAa,GACnC,KAAK,IAAI,aAAa,CAQxB"}