@arizeai/phoenix-evals 2.7.1 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +40 -9
  3. package/dist/esm/decision/decideClassification.d.ts.map +1 -1
  4. package/dist/esm/decision/decideClassification.js +4 -4
  5. package/dist/esm/decision/decideClassification.js.map +1 -1
  6. package/dist/esm/llm/createCompletenessEvaluator.d.ts +1 -1
  7. package/dist/esm/llm/createCompletenessEvaluator.js +1 -1
  8. package/dist/esm/llm/createConcisenessEvaluator.d.ts +1 -1
  9. package/dist/esm/llm/createConcisenessEvaluator.js +1 -1
  10. package/dist/esm/llm/createCorrectnessEvaluator.d.ts +1 -1
  11. package/dist/esm/llm/createCorrectnessEvaluator.js +1 -1
  12. package/dist/esm/llm/createHallucinationEvaluator.d.ts +1 -1
  13. package/dist/esm/llm/createHallucinationEvaluator.js +1 -1
  14. package/dist/esm/llm/createPiiDetectionEvaluator.d.ts +1 -1
  15. package/dist/esm/llm/createPiiDetectionEvaluator.js +1 -1
  16. package/dist/esm/llm/createRefusalEvaluator.d.ts +1 -1
  17. package/dist/esm/llm/createRefusalEvaluator.js +1 -1
  18. package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +1 -1
  19. package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +1 -1
  20. package/dist/esm/llm/createToolInvocationEvaluator.d.ts +1 -1
  21. package/dist/esm/llm/createToolInvocationEvaluator.js +1 -1
  22. package/dist/esm/llm/createToolResponseHandlingEvaluator.d.ts +1 -1
  23. package/dist/esm/llm/createToolResponseHandlingEvaluator.js +1 -1
  24. package/dist/esm/llm/createToolSelectionEvaluator.d.ts +1 -1
  25. package/dist/esm/llm/createToolSelectionEvaluator.js +1 -1
  26. package/dist/esm/llm/createToxicityEvaluator.d.ts +1 -1
  27. package/dist/esm/llm/createToxicityEvaluator.js +1 -1
  28. package/dist/esm/llm/createUserFrictionEvaluator.d.ts +1 -1
  29. package/dist/esm/llm/createUserFrictionEvaluator.js +1 -1
  30. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  31. package/dist/src/decision/decideClassification.d.ts.map +1 -1
  32. package/dist/src/decision/decideClassification.js +4 -4
  33. package/dist/src/decision/decideClassification.js.map +1 -1
  34. package/dist/src/llm/createCompletenessEvaluator.d.ts +1 -1
  35. package/dist/src/llm/createCompletenessEvaluator.js +1 -1
  36. package/dist/src/llm/createConcisenessEvaluator.d.ts +1 -1
  37. package/dist/src/llm/createConcisenessEvaluator.js +1 -1
  38. package/dist/src/llm/createCorrectnessEvaluator.d.ts +1 -1
  39. package/dist/src/llm/createCorrectnessEvaluator.js +1 -1
  40. package/dist/src/llm/createHallucinationEvaluator.d.ts +1 -1
  41. package/dist/src/llm/createHallucinationEvaluator.js +1 -1
  42. package/dist/src/llm/createPiiDetectionEvaluator.d.ts +1 -1
  43. package/dist/src/llm/createPiiDetectionEvaluator.js +1 -1
  44. package/dist/src/llm/createRefusalEvaluator.d.ts +1 -1
  45. package/dist/src/llm/createRefusalEvaluator.js +1 -1
  46. package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +1 -1
  47. package/dist/src/llm/createRetrievalRelevanceEvaluator.js +1 -1
  48. package/dist/src/llm/createToolInvocationEvaluator.d.ts +1 -1
  49. package/dist/src/llm/createToolInvocationEvaluator.js +1 -1
  50. package/dist/src/llm/createToolResponseHandlingEvaluator.d.ts +1 -1
  51. package/dist/src/llm/createToolResponseHandlingEvaluator.js +1 -1
  52. package/dist/src/llm/createToolSelectionEvaluator.d.ts +1 -1
  53. package/dist/src/llm/createToolSelectionEvaluator.js +1 -1
  54. package/dist/src/llm/createToxicityEvaluator.d.ts +1 -1
  55. package/dist/src/llm/createToxicityEvaluator.js +1 -1
  56. package/dist/src/llm/createUserFrictionEvaluator.d.ts +1 -1
  57. package/dist/src/llm/createUserFrictionEvaluator.js +1 -1
  58. package/dist/tsconfig.tsbuildinfo +1 -1
  59. package/docs/classification.mdx +2 -2
  60. package/package.json +9 -9
  61. package/src/decision/decideClassification.ts +4 -5
  62. package/src/llm/createCompletenessEvaluator.ts +1 -1
  63. package/src/llm/createConcisenessEvaluator.ts +1 -1
  64. package/src/llm/createCorrectnessEvaluator.ts +1 -1
  65. package/src/llm/createHallucinationEvaluator.ts +1 -1
  66. package/src/llm/createPiiDetectionEvaluator.ts +1 -1
  67. package/src/llm/createRefusalEvaluator.ts +1 -1
  68. package/src/llm/createRetrievalRelevanceEvaluator.ts +1 -1
  69. package/src/llm/createToolInvocationEvaluator.ts +1 -1
  70. package/src/llm/createToolResponseHandlingEvaluator.ts +1 -1
  71. package/src/llm/createToolSelectionEvaluator.ts +1 -1
  72. package/src/llm/createToxicityEvaluator.ts +1 -1
  73. package/src/llm/createUserFrictionEvaluator.ts +1 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # @arizeai/phoenix-evals
2
2
 
3
+ ## 2.8.0
4
+
5
+ ### Minor Changes
6
+
7
+ - d03cfdc: Support the Gemini Decisions API (`gemini-decision-1`) in classification evaluators. Pass `google.decisionModel("gemini-decision-1")` as the `model` to `createClassificationEvaluator` or any built-in. Requires `@ai-sdk/google` >= 4.0.118.
8
+
3
9
  ## 2.7.1
4
10
 
5
11
  ### Patch Changes
package/README.md CHANGED
@@ -39,11 +39,13 @@ This package is built on [AI SDK](https://ai-sdk.dev/) v7, which is ESM-only. No
39
39
 
40
40
  The library provides a `createClassifier` function that allows you to create custom evaluators for different tasks like hallucination detection, relevance scoring, or any binary/multi-class classification.
41
41
 
42
+ A classifier can be backed by either a **language model** or a **decision model**. A language model returns a label with a written explanation, while a decision model answers through a typed choices API, returning a label with per-label probabilities (and a confidence when the provider reports one) but no explanation.
43
+
42
44
  ```typescript
43
- import { createClassifier } from "@arizeai/phoenix-evals/llm";
45
+ import { createClassificationEvaluator } from "@arizeai/phoenix-evals/llm";
44
46
  import { openai } from "@ai-sdk/openai";
45
47
 
46
- const model = openai("gpt-4o-mini");
48
+ const languageModel = openai("gpt-6-luna");
47
49
 
48
50
  const promptTemplate = `
49
51
  In this task, you will be presented with a query, a reference text and an answer. The answer is
@@ -69,14 +71,15 @@ Is the answer above factual or hallucinated based on the query and reference tex
69
71
  `;
70
72
 
71
73
  // Create the classifier
72
- const evaluator = await createClassifier({
73
- model,
74
+ const evaluator = await createClassificationEvaluator({
75
+ name: "hallucination",
76
+ model: languageModel,
74
77
  choices: { factual: 1, hallucinated: 0 },
75
78
  promptTemplate: promptTemplate,
76
79
  });
77
80
 
78
81
  // Use the classifier
79
- const result = await evaluator({
82
+ const result = await evaluator.evaluate({
80
83
  output: "Arize is not open source.",
81
84
  input: "Is Arize Phoenix Open Source?",
82
85
  reference:
@@ -84,7 +87,35 @@ const result = await evaluator({
84
87
  });
85
88
 
86
89
  console.log(result);
87
- // Output: { label: "hallucinated", score: 0 }
90
+ // Output: { label: "hallucinated", score: 0, explanation: "Arize Phoenix is open source, so the answer is inaccurate." }
91
+
92
+ // The same classifier can be backed by a decision model instead
93
+ const decisionModel = openai.decisionModel("gpt-6-luna");
94
+
95
+ const decisionEvaluator = await createClassificationEvaluator({
96
+ name: "hallucination",
97
+ model: decisionModel,
98
+ choices: { factual: 1, hallucinated: 0 },
99
+ promptTemplate: promptTemplate,
100
+ });
101
+
102
+ const decisionResult = await decisionEvaluator.evaluate({
103
+ output: "Arize is not open source.",
104
+ input: "Is Arize Phoenix Open Source?",
105
+ reference:
106
+ "Arize Phoenix is a platform for building and deploying AI applications. It is open source.",
107
+ });
108
+
109
+ console.log(decisionResult);
110
+ // Output: {
111
+ // label: "hallucinated",
112
+ // score: 0,
113
+ // metadata: {
114
+ // probabilities: { factual: 0.02, hallucinated: 0.98 },
115
+ // confidence: 0.96,
116
+ // modelId: "gpt-6-luna",
117
+ // },
118
+ // }
88
119
  ```
89
120
 
90
121
  See the complete example in [`examples/classifier_example.ts`](examples/classifier_example.ts).
@@ -118,7 +149,7 @@ import {
118
149
  } from "@arizeai/phoenix-evals/llm";
119
150
  import { openai } from "@ai-sdk/openai";
120
151
 
121
- const model = openai("gpt-4o-mini");
152
+ const model = openai("gpt-6-luna");
122
153
 
123
154
  // Faithfulness: checks if the output is grounded in the context
124
155
  const faithfulnessEvaluator = createFaithfulnessEvaluator({ model });
@@ -196,7 +227,7 @@ import { bindEvaluator } from "@arizeai/phoenix-evals";
196
227
  import { createFaithfulnessEvaluator } from "@arizeai/phoenix-evals/llm";
197
228
  import { openai } from "@ai-sdk/openai";
198
229
 
199
- const model = openai("gpt-4o-mini");
230
+ const model = openai("gpt-6-luna");
200
231
 
201
232
  type ExampleType = {
202
233
  question: string;
@@ -250,7 +281,7 @@ import {
250
281
 
251
282
  // Create your evaluator
252
283
  const faithfulnessEvaluator = createFaithfulnessEvaluator({
253
- model: openai("gpt-4o-mini"),
284
+ model: openai("gpt-6-luna"),
254
285
  });
255
286
 
256
287
  // Create a dataset for your experiment
@@ -1 +1 @@
1
- {"version":3,"file":"decideClassification.d.ts","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAe9D;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA2BhC"}
1
+ {"version":3,"file":"decideClassification.d.ts","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAc9D;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA2BhC"}
@@ -11,10 +11,10 @@ import { promptToDecisionState } from "./decisionPrompt.js";
11
11
  */
12
12
  const QUESTION_ID = "label";
13
13
  /**
14
- * The question's instructions. The rendered prompt, rubric included, is sent
15
- * as the state.
14
+ * Instructions sent with the choice question. The OpenAI Decisions API rejects
15
+ * a choice question that has no instructions.
16
16
  */
17
- const INSTRUCTIONS = "Read the prompt in the state and answer it by selecting exactly one of the choices.";
17
+ const CHOICE_INSTRUCTIONS = "Select the single best label for the input.";
18
18
  /**
19
19
  * Classify with an AI SDK decision model by asking one choice question whose
20
20
  * options are the labels. Decision models don't generate text, so the
@@ -32,7 +32,7 @@ export async function decideClassification(args) {
32
32
  questions: {
33
33
  [QUESTION_ID]: {
34
34
  type: "choice",
35
- instructions: INSTRUCTIONS,
35
+ instructions: CHOICE_INSTRUCTIONS,
36
36
  criteria: Object.fromEntries(labels.map((label) => [label, null])),
37
37
  },
38
38
  },
@@ -1 +1 @@
1
- {"version":3,"file":"decideClassification.js","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,IAAI,CAAC;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B;;;GAGG;AACH,MAAM,YAAY,GAChB,qFAAqF,CAAC;AAExF;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAK1C;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;QACvC,KAAK;QACL,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC;QACpC,SAAS,EAAE;YACT,CAAC,WAAW,CAAC,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,YAAY,EAAE,YAAY;gBAC1B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;QACD,SAAS,EAAE,mBAAmB,CAAC;YAC7B,UAAU,EAAE,sBAAsB;YAClC,SAAS;SACV,CAAC;KACH,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,gBAAyB;IAC9C,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,UAAU,GAAI,QAAqD;YACvE,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,OAAO,UAAU,KAAK,QAAQ;YAAE,OAAO,UAAU,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
1
+ {"version":3,"file":"decideClassification.js","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,IAAI,CAAC;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B;;;GAGG;AACH,MAAM,mBAAmB,GAAG,6CAA6C,CAAC;AAE1E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAK1C;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;QACvC,KAAK;QACL,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC;QACpC,SAAS,EAAE;YACT,CAAC,WAAW,CAAC,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,YAAY,EAAE,mBAAmB;gBACjC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;QACD,SAAS,EAAE,mBAAmB,CAAC;YAC7B,UAAU,EAAE,sBAAsB;YAClC,SAAS;SACV,CAAC;KACH,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,gBAAyB;IAC9C,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,UAAU,GAAI,QAAqD;YACvE,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,OAAO,UAAU,KAAK,QAAQ;YAAE,OAAO,UAAU,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
@@ -34,7 +34,7 @@ export type CompletenessEvaluationRecord = {
34
34
  *
35
35
  * @example
36
36
  * ```ts
37
- * const evaluator = createCompletenessEvaluator({ model: openai("gpt-4o-mini") });
37
+ * const evaluator = createCompletenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
38
38
  * const result = await evaluator.evaluate({
39
39
  * conversation:
40
40
  * "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
17
17
  *
18
18
  * @example
19
19
  * ```ts
20
- * const evaluator = createCompletenessEvaluator({ model: openai("gpt-4o-mini") });
20
+ * const evaluator = createCompletenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
21
21
  * const result = await evaluator.evaluate({
22
22
  * conversation:
23
23
  * "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
@@ -31,7 +31,7 @@ export type ConcisenessEvaluationRecord = {
31
31
  *
32
32
  * @example
33
33
  * ```ts
34
- * const evaluator = createConcisenessEvaluator({ model: openai("gpt-4o-mini") });
34
+ * const evaluator = createConcisenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
35
35
  * const result = await evaluator.evaluate({
36
36
  * input: "What is the capital of France?",
37
37
  * output: "Paris.",
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
18
18
  *
19
19
  * @example
20
20
  * ```ts
21
- * const evaluator = createConcisenessEvaluator({ model: openai("gpt-4o-mini") });
21
+ * const evaluator = createConcisenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
22
22
  * const result = await evaluator.evaluate({
23
23
  * input: "What is the capital of France?",
24
24
  * output: "Paris.",
@@ -30,7 +30,7 @@ export type CorrectnessEvaluationRecord = {
30
30
  *
31
31
  * @example
32
32
  * ```ts
33
- * const evaluator = createCorrectnessEvaluator({ model: openai("gpt-4o-mini") });
33
+ * const evaluator = createCorrectnessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
34
34
  * const result = await evaluator.evaluate({
35
35
  * input: "What is the capital of France?",
36
36
  * output: "Paris is the capital of France.",
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
17
17
  *
18
18
  * @example
19
19
  * ```ts
20
- * const evaluator = createCorrectnessEvaluator({ model: openai("gpt-4o-mini") });
20
+ * const evaluator = createCorrectnessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
21
21
  * const result = await evaluator.evaluate({
22
22
  * input: "What is the capital of France?",
23
23
  * output: "Paris is the capital of France.",
@@ -40,7 +40,7 @@ export interface HallucinationEvaluationRecord {
40
40
  *
41
41
  * @example
42
42
  * ```ts
43
- * const evaluator = createHallucinationEvaluator({ model: openai("gpt-4o-mini") });
43
+ * const evaluator = createHallucinationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
44
44
  * const result = await evaluator.evaluate({
45
45
  * input:
46
46
  * "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
@@ -20,7 +20,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
20
20
  *
21
21
  * @example
22
22
  * ```ts
23
- * const evaluator = createHallucinationEvaluator({ model: openai("gpt-4o-mini") });
23
+ * const evaluator = createHallucinationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
24
24
  * const result = await evaluator.evaluate({
25
25
  * input:
26
26
  * "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
@@ -34,7 +34,7 @@ export type PiiDetectionEvaluationRecord = {
34
34
  *
35
35
  * @example
36
36
  * ```ts
37
- * const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
37
+ * const evaluator = createPiiDetectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
38
38
  * const result = await evaluator.evaluate({
39
39
  * conversation:
40
40
  * "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
18
18
  *
19
19
  * @example
20
20
  * ```ts
21
- * const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
21
+ * const evaluator = createPiiDetectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
22
22
  * const result = await evaluator.evaluate({
23
23
  * conversation:
24
24
  * "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
@@ -32,7 +32,7 @@ export type RefusalEvaluationRecord = {
32
32
  *
33
33
  * @example
34
34
  * ```ts
35
- * const evaluator = createRefusalEvaluator({ model: openai("gpt-4o-mini") });
35
+ * const evaluator = createRefusalEvaluator({ model: openai.decisionModel("gpt-6-luna") });
36
36
  * const result = await evaluator.evaluate({
37
37
  * input: "What is the capital of France?",
38
38
  * output: "I'm sorry, I can only help with technical questions.",
@@ -19,7 +19,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
19
19
  *
20
20
  * @example
21
21
  * ```ts
22
- * const evaluator = createRefusalEvaluator({ model: openai("gpt-4o-mini") });
22
+ * const evaluator = createRefusalEvaluator({ model: openai.decisionModel("gpt-6-luna") });
23
23
  * const result = await evaluator.evaluate({
24
24
  * input: "What is the capital of France?",
25
25
  * output: "I'm sorry, I can only help with technical questions.",
@@ -42,7 +42,7 @@ export interface RetrievalRelevanceEvaluationRecord {
42
42
  *
43
43
  * @example
44
44
  * ```ts
45
- * const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
45
+ * const evaluator = createRetrievalRelevanceEvaluator({ model: openai.decisionModel("gpt-6-luna") });
46
46
  * const result = await evaluator.evaluate({
47
47
  * input: "What is the capital of France?",
48
48
  * context: "Paris is the capital and largest city of France.",
@@ -21,7 +21,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
21
21
  *
22
22
  * @example
23
23
  * ```ts
24
- * const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
24
+ * const evaluator = createRetrievalRelevanceEvaluator({ model: openai.decisionModel("gpt-6-luna") });
25
25
  * const result = await evaluator.evaluate({
26
26
  * input: "What is the capital of France?",
27
27
  * context: "Paris is the capital and largest city of France.",
@@ -47,7 +47,7 @@ export type ToolInvocationEvaluationRecord = {
47
47
  *
48
48
  * @example
49
49
  * ```ts
50
- * const evaluator = createToolInvocationEvaluator({ model: openai("gpt-4o-mini") });
50
+ * const evaluator = createToolInvocationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
51
51
  *
52
52
  * // Example with JSON schema format for available tools
53
53
  * const result = await evaluator.evaluate({
@@ -24,7 +24,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
24
24
  *
25
25
  * @example
26
26
  * ```ts
27
- * const evaluator = createToolInvocationEvaluator({ model: openai("gpt-4o-mini") });
27
+ * const evaluator = createToolInvocationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
28
28
  *
29
29
  * // Example with JSON schema format for available tools
30
30
  * const result = await evaluator.evaluate({
@@ -47,7 +47,7 @@ export type ToolResponseHandlingEvaluationRecord = {
47
47
  *
48
48
  * @example
49
49
  * ```ts
50
- * const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-4o-mini") });
50
+ * const evaluator = createToolResponseHandlingEvaluator({ model: openai.decisionModel("gpt-6-luna") });
51
51
  *
52
52
  * // Example: Correct extraction from tool result
53
53
  * const result = await evaluator.evaluate({
@@ -19,7 +19,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
19
19
  *
20
20
  * @example
21
21
  * ```ts
22
- * const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-4o-mini") });
22
+ * const evaluator = createToolResponseHandlingEvaluator({ model: openai.decisionModel("gpt-6-luna") });
23
23
  *
24
24
  * // Example: Correct extraction from tool result
25
25
  * const result = await evaluator.evaluate({
@@ -48,7 +48,7 @@ export type ToolSelectionEvaluationRecord = {
48
48
  *
49
49
  * @example
50
50
  * ```ts
51
- * const evaluator = createToolSelectionEvaluator({ model: openai("gpt-4o-mini") });
51
+ * const evaluator = createToolSelectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
52
52
  *
53
53
  * const result = await evaluator.evaluate({
54
54
  * input: "User: What is the weather in San Francisco?",
@@ -25,7 +25,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
25
25
  *
26
26
  * @example
27
27
  * ```ts
28
- * const evaluator = createToolSelectionEvaluator({ model: openai("gpt-4o-mini") });
28
+ * const evaluator = createToolSelectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
29
29
  *
30
30
  * const result = await evaluator.evaluate({
31
31
  * input: "User: What is the weather in San Francisco?",
@@ -29,7 +29,7 @@ export type ToxicityEvaluationRecord = {
29
29
  *
30
30
  * @example
31
31
  * ```ts
32
- * const evaluator = createToxicityEvaluator({ model: openai("gpt-4o-mini") });
32
+ * const evaluator = createToxicityEvaluator({ model: openai.decisionModel("gpt-6-luna") });
33
33
  * const result = await evaluator.evaluate({
34
34
  * text: "You are a worthless idiot and everyone despises you.",
35
35
  * });
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
17
17
  *
18
18
  * @example
19
19
  * ```ts
20
- * const evaluator = createToxicityEvaluator({ model: openai("gpt-4o-mini") });
20
+ * const evaluator = createToxicityEvaluator({ model: openai.decisionModel("gpt-6-luna") });
21
21
  * const result = await evaluator.evaluate({
22
22
  * text: "You are a worthless idiot and everyone despises you.",
23
23
  * });
@@ -36,7 +36,7 @@ export interface UserFrictionEvaluationRecord {
36
36
  *
37
37
  * @example
38
38
  * ```ts
39
- * const evaluator = createUserFrictionEvaluator({ model: openai("gpt-4o-mini") });
39
+ * const evaluator = createUserFrictionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
40
40
  * const result = await evaluator.evaluate({
41
41
  * conversation: "User: Show recent orders.\nAssistant: Here are last month's orders.",
42
42
  * userMessage: "No, I asked for this week.",
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
18
18
  *
19
19
  * @example
20
20
  * ```ts
21
- * const evaluator = createUserFrictionEvaluator({ model: openai("gpt-4o-mini") });
21
+ * const evaluator = createUserFrictionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
22
22
  * const result = await evaluator.evaluate({
23
23
  * conversation: "User: Show recent orders.\nAssistant: Here are last month's orders.",
24
24
  * userMessage: "No, I asked for this week.",