@arizeai/phoenix-evals 2.5.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/esm/llm/ClassificationEvaluator.d.ts +2 -1
  3. package/dist/esm/llm/ClassificationEvaluator.d.ts.map +1 -1
  4. package/dist/esm/llm/ClassificationEvaluator.js.map +1 -1
  5. package/dist/esm/llm/generateClassification.d.ts +6 -0
  6. package/dist/esm/llm/generateClassification.d.ts.map +1 -1
  7. package/dist/esm/llm/generateClassification.js +66 -1
  8. package/dist/esm/llm/generateClassification.js.map +1 -1
  9. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  10. package/dist/esm/types/evals.d.ts +14 -2
  11. package/dist/esm/types/evals.d.ts.map +1 -1
  12. package/dist/esm/utils/isEvaluationModel.d.ts +13 -0
  13. package/dist/esm/utils/isEvaluationModel.d.ts.map +1 -0
  14. package/dist/esm/utils/isEvaluationModel.js +11 -0
  15. package/dist/esm/utils/isEvaluationModel.js.map +1 -0
  16. package/dist/src/llm/ClassificationEvaluator.d.ts +2 -1
  17. package/dist/src/llm/ClassificationEvaluator.d.ts.map +1 -1
  18. package/dist/src/llm/ClassificationEvaluator.js.map +1 -1
  19. package/dist/src/llm/generateClassification.d.ts +6 -0
  20. package/dist/src/llm/generateClassification.d.ts.map +1 -1
  21. package/dist/src/llm/generateClassification.js +65 -0
  22. package/dist/src/llm/generateClassification.js.map +1 -1
  23. package/dist/src/types/evals.d.ts +14 -2
  24. package/dist/src/types/evals.d.ts.map +1 -1
  25. package/dist/src/utils/isEvaluationModel.d.ts +13 -0
  26. package/dist/src/utils/isEvaluationModel.d.ts.map +1 -0
  27. package/dist/src/utils/isEvaluationModel.js +14 -0
  28. package/dist/src/utils/isEvaluationModel.js.map +1 -0
  29. package/dist/tsconfig.tsbuildinfo +1 -1
  30. package/package.json +11 -10
  31. package/src/llm/ClassificationEvaluator.ts +2 -1
  32. package/src/llm/generateClassification.ts +78 -1
  33. package/src/types/evals.ts +15 -3
  34. package/src/utils/isEvaluationModel.ts +23 -0
@@ -1,4 +1,5 @@
1
1
  import type { LanguageModel } from "ai";
2
+ import type { EvaluationModel } from "../utils/isEvaluationModel.js";
2
3
  import type { ObjectMapping } from "./data.js";
3
4
  import type { WithTelemetry } from "./otel.js";
4
5
  import type { PromptTemplate } from "./templating.js";
@@ -12,7 +13,12 @@ export interface ExampleRecord<OutputType, InputType> {
12
13
  [key: string]: unknown;
13
14
  }
14
15
  export interface WithLLM {
15
- model: LanguageModel;
16
+ /**
17
+ * Either a regular AI SDK {@link LanguageModel}, or an AI SDK
18
+ * {@link EvaluationModel} (e.g. TypeSafe's Jev) which classifies without
19
+ * generating text and therefore cannot produce an explanation.
20
+ */
21
+ model: LanguageModel | EvaluationModel;
16
22
  }
17
23
  export interface LLMEvaluationArgs extends WithLLM {
18
24
  }
@@ -35,6 +41,11 @@ export interface EvaluationResult {
35
41
  * @example "The model correctly identified the sentiment of the text."
36
42
  */
37
43
  explanation?: string;
44
+ /**
45
+ * Additional information about how the result was produced, e.g. the
46
+ * label probabilities reported by an evaluation model.
47
+ */
48
+ metadata?: Record<string, unknown>;
38
49
  }
39
50
  /**
40
51
  * The result of a classification
@@ -42,6 +53,7 @@ export interface EvaluationResult {
42
53
  export interface ClassificationResult {
43
54
  label: string;
44
55
  explanation?: string;
56
+ metadata?: Record<string, unknown>;
45
57
  }
46
58
  /**
47
59
  * The choice (e.g. the label and score mapping) of a classification based evaluation
@@ -58,7 +70,7 @@ export type ClassificationChoicesMap = Record<string, number>;
58
70
  * The arguments for creating a classification-based evaluator
59
71
  */
60
72
  export interface CreateClassifierArgs extends WithTelemetry {
61
- model: LanguageModel;
73
+ model: LanguageModel | EvaluationModel;
62
74
  /**
63
75
  * The choices to classify the example into.
64
76
  * e.g. { "correct": 1, "incorrect": 0 }
@@ -1 +1 @@
1
- {"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAExE;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
1
+ {"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB;;;;OAIG;IACH,KAAK,EAAE,aAAa,GAAG,eAAe,CAAC;CACxC;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,GAAG,eAAe,CAAC;IACvC;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAExE;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
@@ -0,0 +1,13 @@
1
+ import type { Experimental_EvaluationModel, LanguageModel } from "ai";
2
+ /**
3
+ * An AI SDK evaluation model instance (e.g. TypeSafe's Jev). Unlike a
4
+ * {@link LanguageModel}, evaluation models only answer typed questions and
5
+ * cannot generate free-form text.
6
+ */
7
+ export type EvaluationModel = Exclude<Experimental_EvaluationModel, string>;
8
+ /**
9
+ * Type guard that distinguishes an AI SDK evaluation model from a language model.
10
+ * Evaluation models expose `doEvaluate` instead of `doGenerate`.
11
+ */
12
+ export declare function isEvaluationModel(model: LanguageModel | EvaluationModel): model is EvaluationModel;
13
+ //# sourceMappingURL=isEvaluationModel.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"isEvaluationModel.d.ts","sourceRoot":"","sources":["../../../src/utils/isEvaluationModel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,4BAA4B,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEtE;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;AAE5E;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,aAAa,GAAG,eAAe,GACrC,KAAK,IAAI,eAAe,CAO1B"}
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Type guard that distinguishes an AI SDK evaluation model from a language model.
3
+ * Evaluation models expose `doEvaluate` instead of `doGenerate`.
4
+ */
5
+ export function isEvaluationModel(model) {
6
+ return (typeof model === "object" &&
7
+ model !== null &&
8
+ "doEvaluate" in model &&
9
+ typeof model.doEvaluate === "function");
10
+ }
11
+ //# sourceMappingURL=isEvaluationModel.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"isEvaluationModel.js","sourceRoot":"","sources":["../../../src/utils/isEvaluationModel.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAsC;IAEtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,IAAI,KAAK;QACrB,OAAO,KAAK,CAAC,UAAU,KAAK,UAAU,CACvC,CAAC;AACJ,CAAC"}
@@ -1,6 +1,7 @@
1
1
  import type { LanguageModel } from "ai";
2
2
  import type { ClassificationChoicesMap, CreateClassificationEvaluatorArgs, EvaluatorFn, PromptTemplate, WithPromptTemplate } from "../types";
3
3
  import type { ObjectMapping } from "../types/data";
4
+ import type { EvaluationModel } from "../utils/isEvaluationModel";
4
5
  import { LLMEvaluator } from "./LLMEvaluator";
5
6
  /**
6
7
  * An LLM evaluator that performs evaluation via classification
@@ -15,7 +16,7 @@ export declare class ClassificationEvaluator<RecordType extends Record<string, u
15
16
  /**
16
17
  * The model to use for classification
17
18
  */
18
- readonly model: LanguageModel;
19
+ readonly model: LanguageModel | EvaluationModel;
19
20
  /**
20
21
  * The choices to classify the example into
21
22
  */
@@ -1 +1 @@
1
- {"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EACV,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;IAE3C,YAAY,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,EAQ9D;IACD,QAAQ,YAAa,UAAU,kDAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAKrC;CACF"}
1
+ {"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EACV,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,eAAe,CAAC;IAChD;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;IAE3C,YAAY,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC,EAQ9D;IACD,QAAQ,YAAa,UAAU,kDAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAKrC;CACF"}
@@ -1 +1 @@
1
- {"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":";;;AAEA,0CAAmD;AASnD,oEAA0D;AAC1D,6DAA0D;AAC1D,iDAA8C;AAE9C;;GAEG;AACH,6BACE,SAAQ,2BAAwB;IAkBhC,YAAY,IAAmD;QAC7D,KAAK,CAAC,IAAI,CAAC,CAAC;QAQd,aAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;YACjC,OAAO,IAAI,CAAC,WAAW,CACrB,IAAI,CAAC,YAAY;gBACf,CAAC,CAAC,IAAA,gCAAW,EAAa,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;gBACrD,CAAC,CAAC,OAAO,CACZ,CAAC;QACJ,CAAC,CAAC;QAbA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAA,uCAAkB,oBAChC,IAAI,EACP,CAAC;IACL,CAAC;IAQD;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,IAAA,+BAAoB,EAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,uBAAuB,iCAC7B,IAAI,KACP,YAAY,IACZ,CAAC;IACL,CAAC;CACF"}
1
+ {"version":3,"file":"ClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":";;;AAEA,0CAAmD;AAUnD,oEAA0D;AAC1D,6DAA0D;AAC1D,iDAA8C;AAE9C;;GAEG;AACH,6BACE,SAAQ,2BAAwB;IAkBhC,YAAY,IAAmD;QAC7D,KAAK,CAAC,IAAI,CAAC,CAAC;QAQd,aAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE;YACjC,OAAO,IAAI,CAAC,WAAW,CACrB,IAAI,CAAC,YAAY;gBACf,CAAC,CAAC,IAAA,gCAAW,EAAa,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC;gBACrD,CAAC,CAAC,OAAO,CACZ,CAAC;QACJ,CAAC,CAAC;QAbA,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAA,uCAAkB,oBAChC,IAAI,EACP,CAAC;IACL,CAAC;IAQD;;OAEG;IACH,IAAI,uBAAuB;QACzB,0DAA0D;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,wBAAwB,GAAG,IAAA,+BAAoB,EAAC;gBACnD,QAAQ,EAAE,IAAI,CAAC,cAAc;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,+BAA+B;QAC/B,OAAO,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAuC;QAEvC,OAAO,IAAI,uBAAuB,iCAC7B,IAAI,KACP,YAAY,IACZ,CAAC;IACL,CAAC;CACF"}
@@ -17,6 +17,12 @@ export type ClassifyArgs = WithLLM & WithTelemetry & WithPrompt & {
17
17
  };
18
18
  /**
19
19
  * A function that leverages an llm to perform a classification
20
+ *
21
+ * When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
22
+ * classification is routed through `experimental_evaluate` as a single
23
+ * choice question. Evaluation models cannot generate text, so no explanation
24
+ * is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
25
+ * are ignored on that path: `experimental_evaluate` does not emit spans yet.
20
26
  */
21
27
  export declare function generateClassification(args: ClassifyArgs): Promise<ClassificationResult>;
22
28
  //# sourceMappingURL=generateClassification.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CA4B/B"}
1
+ {"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAMnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAgC/B"}
@@ -15,12 +15,22 @@ exports.generateClassification = generateClassification;
15
15
  const ai_1 = require("ai");
16
16
  const zod_1 = require("zod");
17
17
  const telemetry_1 = require("../telemetry");
18
+ const isEvaluationModel_1 = require("../utils/isEvaluationModel");
18
19
  /**
19
20
  * A function that leverages an llm to perform a classification
21
+ *
22
+ * When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
23
+ * classification is routed through `experimental_evaluate` as a single
24
+ * choice question. Evaluation models cannot generate text, so no explanation
25
+ * is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
26
+ * are ignored on that path: `experimental_evaluate` does not emit spans yet.
20
27
  */
21
28
  async function generateClassification(args) {
22
29
  var _a, _b;
23
30
  const { labels, model, schemaName, schemaDescription, telemetry } = args, prompt = __rest(args, ["labels", "model", "schemaName", "schemaDescription", "telemetry"]);
31
+ if ((0, isEvaluationModel_1.isEvaluationModel)(model)) {
32
+ return evaluateClassification({ model, labels, prompt });
33
+ }
24
34
  const telemetryOptions = {
25
35
  isEnabled: (_a = telemetry === null || telemetry === void 0 ? void 0 : telemetry.isEnabled) !== null && _a !== void 0 ? _a : true,
26
36
  functionId: "generateClassification",
@@ -40,4 +50,59 @@ async function generateClassification(args) {
40
50
  explanation: result.object.explanation,
41
51
  };
42
52
  }
53
+ /**
54
+ * Classify using an AI SDK evaluation model. The rendered prompt becomes the
55
+ * shared evaluation state and the labels become the options of one choice
56
+ * question. The model's label probabilities and resolved model id are
57
+ * returned as metadata.
58
+ */
59
+ async function evaluateClassification(args) {
60
+ const { model, labels, prompt } = args;
61
+ const result = await (0, ai_1.experimental_evaluate)({
62
+ model,
63
+ state: toEvaluationState(prompt),
64
+ questions: {
65
+ label: {
66
+ type: "choice",
67
+ instructions: "Read the prompt in the state and answer it by selecting exactly one of the choices.",
68
+ criteria: Object.fromEntries(labels.map((label) => [label, null])),
69
+ },
70
+ },
71
+ });
72
+ const answer = result.answers.label;
73
+ return {
74
+ label: answer.choice,
75
+ metadata: {
76
+ probabilities: answer.probabilities,
77
+ modelId: result.response.modelId,
78
+ },
79
+ };
80
+ }
81
+ /**
82
+ * Convert an AI SDK prompt into JSON-compatible evaluation state.
83
+ * A bare text prompt is passed through as a string; anything else
84
+ * (instructions, system messages, message arrays) is serialized to JSON.
85
+ */
86
+ function toEvaluationState(prompt) {
87
+ const { instructions, system, messages, prompt: promptContent } = prompt;
88
+ if (typeof promptContent === "string" &&
89
+ instructions === undefined &&
90
+ system === undefined &&
91
+ messages === undefined) {
92
+ return promptContent;
93
+ }
94
+ const state = {};
95
+ if (instructions !== undefined)
96
+ state.instructions = instructions;
97
+ if (system !== undefined)
98
+ state.system = system;
99
+ if (promptContent !== undefined)
100
+ state.prompt = promptContent;
101
+ if (messages !== undefined)
102
+ state.messages = messages;
103
+ // Round-trip through JSON so the state only contains JSON values
104
+ // (strips `undefined`, turns Date/URL into strings). Binary file parts
105
+ // are not supported by evaluation models and are not stripped here.
106
+ return JSON.parse(JSON.stringify(state));
107
+ }
43
108
  //# sourceMappingURL=generateClassification.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,2BAAoC;AACpC,6BAAwB;AAExB,4CAAgE;AAoBhE;;GAEG;AACI,KAAK,iCACV,IAAkB;;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,KAC7D,IAAI,EAD8D,MAAM,UACxE,IAAI,EADA,mEAAsE,CACtE,CAAC;IAEP,MAAM,gBAAgB,GAAG;QACvB,SAAS,QAAE,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,SAAS,mCAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,YAAY,EAAE,IAAA,oCAAwB,QAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,mCAAI,kBAAM,CAAC;KACpE,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAc,kBACjC,KAAK;QACL,UAAU;QACV,iBAAiB,EACjB,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC,EACF,SAAS,EAAE,gBAAgB;QAC3B,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI,IACxB,MAAM,EACT,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,2BAA2D;AAC3D,6BAAwB;AAExB,4CAAgE;AAIhE,kEAGoC;AAkBpC;;;;;;;;GAQG;AACI,KAAK,iCACV,IAAkB;;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,KAC7D,IAAI,EAD8D,MAAM,UACxE,IAAI,EADA,mEAAsE,CACtE,CAAC;IAEP,IAAI,IAAA,qCAAiB,EAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,sBAAsB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,gBAAgB,GAAG;QACvB,SAAS,QAAE,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,SAAS,mCAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,YAAY,EAAE,IAAA,oCAAwB,QAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,mCAAI,kBAAM,CAAC;KACpE,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAc,kBACjC,KAAK;QACL,UAAU;QACV,iBAAiB,EACjB,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC,EACF,SAAS,EAAE,gBAAgB;QAC3B,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI,IACxB,MAAM,EACT,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC;AAID;;;;;GAKG;AACH,KAAK,UAAU,sBAAsB,CAAC,IAIrC;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAA,0BAAqB,EAAC;QACzC,KAAK;QACL,KAAK,EAAE,iBAAiB,CAAC,MAAM,CAAC;QAChC,SAAS,EAAE;YACT,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,YAAY,EACV,qFAAqF;gBACvF,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;KACF,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;IACpC,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,MAAkB;IAC3C,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,CAAC;IACzE,IACE,OAAO,aAAa,KAAK,QAAQ;QACjC,YAAY,KAAK,SAAS;QAC1B,MAAM,KAAK,SAAS;QACpB,QAAQ,KAAK,SAAS,EACtB,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,IAAI,YAAY,KAAK,SAAS;QAAE,KAAK,CAAC,YAAY,GAAG,YAAY,CAAC;IAClE,IAAI,MAAM,KAAK,SAAS;QAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IAChD,IAAI,aAAa,KAAK,SAAS;QAAE,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC;IAC9D,IAAI,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACtD,iEAAiE;IACjE,uEAAuE;IACvE,oEAAoE;IACpE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAoB,CAAC;AAC9D,CAAC"}
@@ -1,4 +1,5 @@
1
1
  import type { LanguageModel } from "ai";
2
+ import type { EvaluationModel } from "../utils/isEvaluationModel";
2
3
  import type { ObjectMapping } from "./data";
3
4
  import type { WithTelemetry } from "./otel";
4
5
  import type { PromptTemplate } from "./templating";
@@ -12,7 +13,12 @@ export interface ExampleRecord<OutputType, InputType> {
12
13
  [key: string]: unknown;
13
14
  }
14
15
  export interface WithLLM {
15
- model: LanguageModel;
16
+ /**
17
+ * Either a regular AI SDK {@link LanguageModel}, or an AI SDK
18
+ * {@link EvaluationModel} (e.g. TypeSafe's Jev) which classifies without
19
+ * generating text and therefore cannot produce an explanation.
20
+ */
21
+ model: LanguageModel | EvaluationModel;
16
22
  }
17
23
  export interface LLMEvaluationArgs extends WithLLM {
18
24
  }
@@ -35,6 +41,11 @@ export interface EvaluationResult {
35
41
  * @example "The model correctly identified the sentiment of the text."
36
42
  */
37
43
  explanation?: string;
44
+ /**
45
+ * Additional information about how the result was produced, e.g. the
46
+ * label probabilities reported by an evaluation model.
47
+ */
48
+ metadata?: Record<string, unknown>;
38
49
  }
39
50
  /**
40
51
  * The result of a classification
@@ -42,6 +53,7 @@ export interface EvaluationResult {
42
53
  export interface ClassificationResult {
43
54
  label: string;
44
55
  explanation?: string;
56
+ metadata?: Record<string, unknown>;
45
57
  }
46
58
  /**
47
59
  * The choice (e.g. the label and score mapping) of a classification based evaluation
@@ -58,7 +70,7 @@ export type ClassificationChoicesMap = Record<string, number>;
58
70
  * The arguments for creating a classification-based evaluator
59
71
  */
60
72
  export interface CreateClassifierArgs extends WithTelemetry {
61
- model: LanguageModel;
73
+ model: LanguageModel | EvaluationModel;
62
74
  /**
63
75
  * The choices to classify the example into.
64
76
  * e.g. { "correct": 1, "incorrect": 0 }
@@ -1 +1 @@
1
- {"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAExE;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
1
+ {"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB;;;;OAIG;IACH,KAAK,EAAE,aAAa,GAAG,eAAe,CAAC;CACxC;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,GAAG,eAAe,CAAC;IACvC;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAExE;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
@@ -0,0 +1,13 @@
1
+ import type { Experimental_EvaluationModel, LanguageModel } from "ai";
2
+ /**
3
+ * An AI SDK evaluation model instance (e.g. TypeSafe's Jev). Unlike a
4
+ * {@link LanguageModel}, evaluation models only answer typed questions and
5
+ * cannot generate free-form text.
6
+ */
7
+ export type EvaluationModel = Exclude<Experimental_EvaluationModel, string>;
8
+ /**
9
+ * Type guard that distinguishes an AI SDK evaluation model from a language model.
10
+ * Evaluation models expose `doEvaluate` instead of `doGenerate`.
11
+ */
12
+ export declare function isEvaluationModel(model: LanguageModel | EvaluationModel): model is EvaluationModel;
13
+ //# sourceMappingURL=isEvaluationModel.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"isEvaluationModel.d.ts","sourceRoot":"","sources":["../../../src/utils/isEvaluationModel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,4BAA4B,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEtE;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;AAE5E;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,aAAa,GAAG,eAAe,GACrC,KAAK,IAAI,eAAe,CAO1B"}
@@ -0,0 +1,14 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.isEvaluationModel = isEvaluationModel;
4
+ /**
5
+ * Type guard that distinguishes an AI SDK evaluation model from a language model.
6
+ * Evaluation models expose `doEvaluate` instead of `doGenerate`.
7
+ */
8
+ function isEvaluationModel(model) {
9
+ return (typeof model === "object" &&
10
+ model !== null &&
11
+ "doEvaluate" in model &&
12
+ typeof model.doEvaluate === "function");
13
+ }
14
+ //# sourceMappingURL=isEvaluationModel.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"isEvaluationModel.js","sourceRoot":"","sources":["../../../src/utils/isEvaluationModel.ts"],"names":[],"mappings":";;;AASA;;;GAGG;AACH,2BACE,KAAsC;IAEtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,IAAI,KAAK;QACrB,OAAO,KAAK,CAAC,UAAU,KAAK,UAAU,CACvC,CAAC;AACJ,CAAC"}