@arizeai/phoenix-evals 2.4.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CHANGELOG.md +262 -0
  2. package/README.md +6 -5
  3. package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
  4. package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
  5. package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js +93 -0
  6. package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
  7. package/dist/esm/__generated__/default_templates/index.d.ts +1 -0
  8. package/dist/esm/__generated__/default_templates/index.d.ts.map +1 -1
  9. package/dist/esm/__generated__/default_templates/index.js +1 -0
  10. package/dist/esm/__generated__/default_templates/index.js.map +1 -1
  11. package/dist/esm/llm/ClassificationEvaluator.d.ts +2 -1
  12. package/dist/esm/llm/ClassificationEvaluator.d.ts.map +1 -1
  13. package/dist/esm/llm/ClassificationEvaluator.js.map +1 -1
  14. package/dist/esm/llm/createCompletenessEvaluator.d.ts +46 -0
  15. package/dist/esm/llm/createCompletenessEvaluator.d.ts.map +1 -0
  16. package/dist/esm/llm/createCompletenessEvaluator.js +38 -0
  17. package/dist/esm/llm/createCompletenessEvaluator.js.map +1 -0
  18. package/dist/esm/llm/createDocumentRelevanceEvaluator.d.ts +16 -11
  19. package/dist/esm/llm/createDocumentRelevanceEvaluator.d.ts.map +1 -1
  20. package/dist/esm/llm/createDocumentRelevanceEvaluator.js +7 -11
  21. package/dist/esm/llm/createDocumentRelevanceEvaluator.js.map +1 -1
  22. package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +4 -5
  23. package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -1
  24. package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +4 -5
  25. package/dist/esm/llm/createRetrievalRelevanceEvaluator.js.map +1 -1
  26. package/dist/esm/llm/generateClassification.d.ts +6 -0
  27. package/dist/esm/llm/generateClassification.d.ts.map +1 -1
  28. package/dist/esm/llm/generateClassification.js +66 -1
  29. package/dist/esm/llm/generateClassification.js.map +1 -1
  30. package/dist/esm/llm/index.d.ts +1 -0
  31. package/dist/esm/llm/index.d.ts.map +1 -1
  32. package/dist/esm/llm/index.js +1 -0
  33. package/dist/esm/llm/index.js.map +1 -1
  34. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  35. package/dist/esm/types/evals.d.ts +14 -2
  36. package/dist/esm/types/evals.d.ts.map +1 -1
  37. package/dist/esm/utils/bindEvaluator.d.ts +4 -5
  38. package/dist/esm/utils/bindEvaluator.d.ts.map +1 -1
  39. package/dist/esm/utils/bindEvaluator.js +4 -5
  40. package/dist/esm/utils/bindEvaluator.js.map +1 -1
  41. package/dist/esm/utils/isEvaluationModel.d.ts +13 -0
  42. package/dist/esm/utils/isEvaluationModel.d.ts.map +1 -0
  43. package/dist/esm/utils/isEvaluationModel.js +11 -0
  44. package/dist/esm/utils/isEvaluationModel.js.map +1 -0
  45. package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
  46. package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
  47. package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js +96 -0
  48. package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
  49. package/dist/src/__generated__/default_templates/index.d.ts +1 -0
  50. package/dist/src/__generated__/default_templates/index.d.ts.map +1 -1
  51. package/dist/src/__generated__/default_templates/index.js +3 -1
  52. package/dist/src/__generated__/default_templates/index.js.map +1 -1
  53. package/dist/src/llm/ClassificationEvaluator.d.ts +2 -1
  54. package/dist/src/llm/ClassificationEvaluator.d.ts.map +1 -1
  55. package/dist/src/llm/ClassificationEvaluator.js.map +1 -1
  56. package/dist/src/llm/createCompletenessEvaluator.d.ts +46 -0
  57. package/dist/src/llm/createCompletenessEvaluator.d.ts.map +1 -0
  58. package/dist/src/llm/createCompletenessEvaluator.js +49 -0
  59. package/dist/src/llm/createCompletenessEvaluator.js.map +1 -0
  60. package/dist/src/llm/createDocumentRelevanceEvaluator.d.ts +16 -11
  61. package/dist/src/llm/createDocumentRelevanceEvaluator.d.ts.map +1 -1
  62. package/dist/src/llm/createDocumentRelevanceEvaluator.js +7 -11
  63. package/dist/src/llm/createDocumentRelevanceEvaluator.js.map +1 -1
  64. package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +4 -5
  65. package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -1
  66. package/dist/src/llm/createRetrievalRelevanceEvaluator.js +4 -5
  67. package/dist/src/llm/createRetrievalRelevanceEvaluator.js.map +1 -1
  68. package/dist/src/llm/generateClassification.d.ts +6 -0
  69. package/dist/src/llm/generateClassification.d.ts.map +1 -1
  70. package/dist/src/llm/generateClassification.js +65 -0
  71. package/dist/src/llm/generateClassification.js.map +1 -1
  72. package/dist/src/llm/index.d.ts +1 -0
  73. package/dist/src/llm/index.d.ts.map +1 -1
  74. package/dist/src/llm/index.js +1 -0
  75. package/dist/src/llm/index.js.map +1 -1
  76. package/dist/src/types/evals.d.ts +14 -2
  77. package/dist/src/types/evals.d.ts.map +1 -1
  78. package/dist/src/utils/bindEvaluator.d.ts +4 -5
  79. package/dist/src/utils/bindEvaluator.d.ts.map +1 -1
  80. package/dist/src/utils/bindEvaluator.js +4 -5
  81. package/dist/src/utils/bindEvaluator.js.map +1 -1
  82. package/dist/src/utils/isEvaluationModel.d.ts +13 -0
  83. package/dist/src/utils/isEvaluationModel.d.ts.map +1 -0
  84. package/dist/src/utils/isEvaluationModel.js +14 -0
  85. package/dist/src/utils/isEvaluationModel.js.map +1 -0
  86. package/dist/tsconfig.tsbuildinfo +1 -1
  87. package/package.json +13 -12
  88. package/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.ts +95 -0
  89. package/src/__generated__/default_templates/index.ts +1 -0
  90. package/src/llm/ClassificationEvaluator.ts +2 -1
  91. package/src/llm/createCompletenessEvaluator.ts +74 -0
  92. package/src/llm/createDocumentRelevanceEvaluator.ts +16 -11
  93. package/src/llm/createRetrievalRelevanceEvaluator.ts +4 -5
  94. package/src/llm/generateClassification.ts +78 -1
  95. package/src/llm/index.ts +1 -0
  96. package/src/types/evals.ts +15 -3
  97. package/src/utils/bindEvaluator.ts +4 -5
  98. package/src/utils/isEvaluationModel.ts +23 -0
@@ -3,6 +3,12 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
3
3
  /**
4
4
  * Creates a document relevance evaluator function.
5
5
  *
6
+ * @deprecated Use {@link createRetrievalRelevanceEvaluator} instead. Rename the
7
+ * `documentText` input field to `context`, and update code that checks for the
8
+ * `unrelated` label to check for `irrelevant`. Pass one document as `context` to
9
+ * preserve per-document evaluation. This function will be removed in the next major
10
+ * release.
11
+ *
6
12
  * This function returns an evaluator that determines whether a given document text
7
13
  * is relevant to a provided input question. The evaluator uses a classification model
8
14
  * and a prompt template to make its determination.
@@ -13,18 +19,8 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
13
19
  * @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCE_TEMPLATE).
14
20
  * @param args.telemetry - The telemetry to use for the evaluator.
15
21
  *
16
- * @returns An evaluator function that takes a {@link DocumentRelevanceExample} and returns a classification result
22
+ * @returns An evaluator function that takes a {@link DocumentRelevanceEvaluationRecord} and returns a classification result
17
23
  * indicating whether the document is relevant to the input question.
18
- *
19
- * @example
20
- * ```ts
21
- * const evaluator = createDocumentRelevanceEvaluator({ model: openai("gpt-4o-mini") });
22
- * const result = await evaluator.evaluate({
23
- * input: "What is the capital of France?",
24
- * documentText: "Paris is the capital and most populous city of France.",
25
- * });
26
- * console.log(result.label); // "relevant" or "unrelated"
27
- * ```
28
24
  */
29
25
  export function createDocumentRelevanceEvaluator(args) {
30
26
  const { choices = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
@@ -1 +1 @@
1
- {"version":3,"file":"createDocumentRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kDAAkD,EAAE,MAAM,oCAAoC,CAAC;AAGxG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAwBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,gCAAgC,CAI9C,IAAgD;IAEhD,MAAM,EACJ,OAAO,GAAG,kDAAkD,CAAC,OAAO,EACpE,cAAc,GAAG,kDAAkD,CAAC,QAAQ,EAC5E,qBAAqB,GAAG,kDAAkD,CAAC,qBAAqB,EAChG,IAAI,GAAG,kDAAkD,CAAC,IAAI,EAC9D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"createDocumentRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kDAAkD,EAAE,MAAM,oCAAoC,CAAC;AAGxG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAiChF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,gCAAgC,CAI9C,IAAgD;IAEhD,MAAM,EACJ,OAAO,GAAG,kDAAkD,CAAC,OAAO,EACpE,cAAc,GAAG,kDAAkD,CAAC,QAAQ,EAC5E,qBAAqB,GAAG,kDAAkD,CAAC,qBAAqB,EAChG,IAAI,GAAG,kDAAkD,CAAC,IAAI,EAC9D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
@@ -26,11 +26,10 @@ export interface RetrievalRelevanceEvaluationRecord {
26
26
  *
27
27
  * This function returns an evaluator that determines whether the external
28
28
  * information retrieved during a step is relevant to the request it was
29
- * serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
30
- * single document against a question, this evaluator is source-agnostic and
31
- * scores the retrieved information as a whole (holistically, per retrieval
32
- * step), whether it came from a vector search, a tool or MCP call, a web
33
- * search, or content embedded in an LLM turn.
29
+ * serving. It is source-agnostic and scores the retrieved information as a
30
+ * whole, whether it came from a vector search, a tool or MCP call, a web search,
31
+ * or content embedded in an LLM turn. Pass one document as `context` to evaluate
32
+ * documents individually.
34
33
  *
35
34
  * @param args - The arguments for creating the retrieval relevance evaluator.
36
35
  * @param args.model - The model to use for classification.
@@ -1 +1 @@
1
- {"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
1
+ {"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
@@ -5,11 +5,10 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
5
5
  *
6
6
  * This function returns an evaluator that determines whether the external
7
7
  * information retrieved during a step is relevant to the request it was
8
- * serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
9
- * single document against a question, this evaluator is source-agnostic and
10
- * scores the retrieved information as a whole (holistically, per retrieval
11
- * step), whether it came from a vector search, a tool or MCP call, a web
12
- * search, or content embedded in an LLM turn.
8
+ * serving. It is source-agnostic and scores the retrieved information as a
9
+ * whole, whether it came from a vector search, a tool or MCP call, a web search,
10
+ * or content embedded in an LLM turn. Pass one document as `context` to evaluate
11
+ * documents individually.
13
12
  *
14
13
  * @param args - The arguments for creating the retrieval relevance evaluator.
15
14
  * @param args.model - The model to use for classification.
@@ -1 +1 @@
1
- {"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,UAAU,iCAAiC,CAI/C,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,mDAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,mDAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,mDAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,mDAAmD,CAAC,IAAI,EAC/D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,UAAU,iCAAiC,CAI/C,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,mDAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,mDAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,mDAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,mDAAmD,CAAC,IAAI,EAC/D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
@@ -17,6 +17,12 @@ export type ClassifyArgs = WithLLM & WithTelemetry & WithPrompt & {
17
17
  };
18
18
  /**
19
19
  * A function that leverages an llm to perform a classification
20
+ *
21
+ * When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
22
+ * classification is routed through `experimental_evaluate` as a single
23
+ * choice question. Evaluation models cannot generate text, so no explanation
24
+ * is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
25
+ * are ignored on that path: `experimental_evaluate` does not emit spans yet.
20
26
  */
21
27
  export declare function generateClassification(args: ClassifyArgs): Promise<ClassificationResult>;
22
28
  //# sourceMappingURL=generateClassification.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CA4B/B"}
1
+ {"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAMnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAgC/B"}
@@ -1,11 +1,21 @@
1
- import { generateObject } from "ai";
1
+ import { experimental_evaluate, generateObject } from "ai";
2
2
  import { z } from "zod";
3
3
  import { getTelemetryIntegrations, tracer } from "../telemetry/index.js";
4
+ import { isEvaluationModel, } from "../utils/isEvaluationModel.js";
4
5
  /**
5
6
  * A function that leverages an llm to perform a classification
7
+ *
8
+ * When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
9
+ * classification is routed through `experimental_evaluate` as a single
10
+ * choice question. Evaluation models cannot generate text, so no explanation
11
+ * is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
12
+ * are ignored on that path: `experimental_evaluate` does not emit spans yet.
6
13
  */
7
14
  export async function generateClassification(args) {
8
15
  const { labels, model, schemaName, schemaDescription, telemetry, ...prompt } = args;
16
+ if (isEvaluationModel(model)) {
17
+ return evaluateClassification({ model, labels, prompt });
18
+ }
9
19
  const telemetryOptions = {
10
20
  isEnabled: telemetry?.isEnabled ?? true,
11
21
  functionId: "generateClassification",
@@ -30,4 +40,59 @@ export async function generateClassification(args) {
30
40
  explanation: result.object.explanation,
31
41
  };
32
42
  }
43
+ /**
44
+ * Classify using an AI SDK evaluation model. The rendered prompt becomes the
45
+ * shared evaluation state and the labels become the options of one choice
46
+ * question. The model's label probabilities and resolved model id are
47
+ * returned as metadata.
48
+ */
49
+ async function evaluateClassification(args) {
50
+ const { model, labels, prompt } = args;
51
+ const result = await experimental_evaluate({
52
+ model,
53
+ state: toEvaluationState(prompt),
54
+ questions: {
55
+ label: {
56
+ type: "choice",
57
+ instructions: "Read the prompt in the state and answer it by selecting exactly one of the choices.",
58
+ criteria: Object.fromEntries(labels.map((label) => [label, null])),
59
+ },
60
+ },
61
+ });
62
+ const answer = result.answers.label;
63
+ return {
64
+ label: answer.choice,
65
+ metadata: {
66
+ probabilities: answer.probabilities,
67
+ modelId: result.response.modelId,
68
+ },
69
+ };
70
+ }
71
+ /**
72
+ * Convert an AI SDK prompt into JSON-compatible evaluation state.
73
+ * A bare text prompt is passed through as a string; anything else
74
+ * (instructions, system messages, message arrays) is serialized to JSON.
75
+ */
76
+ function toEvaluationState(prompt) {
77
+ const { instructions, system, messages, prompt: promptContent } = prompt;
78
+ if (typeof promptContent === "string" &&
79
+ instructions === undefined &&
80
+ system === undefined &&
81
+ messages === undefined) {
82
+ return promptContent;
83
+ }
84
+ const state = {};
85
+ if (instructions !== undefined)
86
+ state.instructions = instructions;
87
+ if (system !== undefined)
88
+ state.system = system;
89
+ if (promptContent !== undefined)
90
+ state.prompt = promptContent;
91
+ if (messages !== undefined)
92
+ state.messages = messages;
93
+ // Round-trip through JSON so the state only contains JSON values
94
+ // (strips `undefined`, turns Date/URL into strings). Binary file parts
95
+ // are not supported by evaluation models and are not stripped here.
96
+ return JSON.parse(JSON.stringify(state));
97
+ }
33
98
  //# sourceMappingURL=generateClassification.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,wBAAwB,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAoBhE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAC1E,IAAI,CAAC;IAEP,MAAM,gBAAgB,GAAG;QACvB,SAAS,EAAE,SAAS,EAAE,SAAS,IAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,YAAY,EAAE,wBAAwB,CAAC,SAAS,EAAE,MAAM,IAAI,MAAM,CAAC;KACpE,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,UAAU;QACV,iBAAiB;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC;QACF,SAAS,EAAE,gBAAgB;QAC3B,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI;QAC3B,GAAG,MAAM;KACV,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AAC3D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,wBAAwB,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAIhE,OAAO,EAEL,iBAAiB,GAClB,MAAM,4BAA4B,CAAC;AAkBpC;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAC1E,IAAI,CAAC;IAEP,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,sBAAsB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,gBAAgB,GAAG;QACvB,SAAS,EAAE,SAAS,EAAE,SAAS,IAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,YAAY,EAAE,wBAAwB,CAAC,SAAS,EAAE,MAAM,IAAI,MAAM,CAAC;KACpE,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,UAAU;QACV,iBAAiB;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC;QACF,SAAS,EAAE,gBAAgB;QAC3B,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI;QAC3B,GAAG,MAAM;KACV,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC;AAID;;;;;GAKG;AACH,KAAK,UAAU,sBAAsB,CAAC,IAIrC;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC;QACzC,KAAK;QACL,KAAK,EAAE,iBAAiB,CAAC,MAAM,CAAC;QAChC,SAAS,EAAE;YACT,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,YAAY,EACV,qFAAqF;gBACvF,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;KACF,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;IACpC,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,MAAkB;IAC3C,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,CAAC;IACzE,IACE,OAAO,aAAa,KAAK,QAAQ;QACjC,YAAY,KAAK,SAAS;QAC1B,MAAM,KAAK,SAAS;QACpB,QAAQ,KAAK,SAAS,EACtB,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,IAAI,YAAY,KAAK,SAAS;QAAE,KAAK,CAAC,YAAY,GAAG,YAAY,CAAC;IAClE,IAAI,MAAM,KAAK,SAAS;QAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IAChD,IAAI,aAAa,KAAK,SAAS;QAAE,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC;IAC9D,IAAI,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACtD,iEAAiE;IACjE,uEAAuE;IACvE,oEAAoE;IACpE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAoB,CAAC;AAC9D,CAAC"}
@@ -1,6 +1,7 @@
1
1
  export * from "./ClassificationEvaluator.js";
2
2
  export * from "./createClassificationEvaluator.js";
3
3
  export * from "./createClassifierFn.js";
4
+ export * from "./createCompletenessEvaluator.js";
4
5
  export * from "./createConcisenessEvaluator.js";
5
6
  export * from "./createCorrectnessEvaluator.js";
6
7
  export * from "./createDocumentRelevanceEvaluator.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
@@ -1,6 +1,7 @@
1
1
  export * from "./ClassificationEvaluator.js";
2
2
  export * from "./createClassificationEvaluator.js";
3
3
  export * from "./createClassifierFn.js";
4
+ export * from "./createCompletenessEvaluator.js";
4
5
  export * from "./createConcisenessEvaluator.js";
5
6
  export * from "./createCorrectnessEvaluator.js";
6
7
  export * from "./createDocumentRelevanceEvaluator.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}