@arizeai/phoenix-evals 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +262 -0
- package/README.md +6 -5
- package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js +93 -0
- package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/esm/__generated__/default_templates/index.d.ts +1 -0
- package/dist/esm/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/esm/__generated__/default_templates/index.js +1 -0
- package/dist/esm/__generated__/default_templates/index.js.map +1 -1
- package/dist/esm/llm/ClassificationEvaluator.d.ts +2 -1
- package/dist/esm/llm/ClassificationEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/ClassificationEvaluator.js.map +1 -1
- package/dist/esm/llm/createCompletenessEvaluator.d.ts +46 -0
- package/dist/esm/llm/createCompletenessEvaluator.d.ts.map +1 -0
- package/dist/esm/llm/createCompletenessEvaluator.js +38 -0
- package/dist/esm/llm/createCompletenessEvaluator.js.map +1 -0
- package/dist/esm/llm/createDocumentRelevanceEvaluator.d.ts +16 -11
- package/dist/esm/llm/createDocumentRelevanceEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/createDocumentRelevanceEvaluator.js +7 -11
- package/dist/esm/llm/createDocumentRelevanceEvaluator.js.map +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +4 -5
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +4 -5
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js.map +1 -1
- package/dist/esm/llm/generateClassification.d.ts +6 -0
- package/dist/esm/llm/generateClassification.d.ts.map +1 -1
- package/dist/esm/llm/generateClassification.js +66 -1
- package/dist/esm/llm/generateClassification.js.map +1 -1
- package/dist/esm/llm/index.d.ts +1 -0
- package/dist/esm/llm/index.d.ts.map +1 -1
- package/dist/esm/llm/index.js +1 -0
- package/dist/esm/llm/index.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/esm/types/evals.d.ts +14 -2
- package/dist/esm/types/evals.d.ts.map +1 -1
- package/dist/esm/utils/bindEvaluator.d.ts +4 -5
- package/dist/esm/utils/bindEvaluator.d.ts.map +1 -1
- package/dist/esm/utils/bindEvaluator.js +4 -5
- package/dist/esm/utils/bindEvaluator.js.map +1 -1
- package/dist/esm/utils/isEvaluationModel.d.ts +13 -0
- package/dist/esm/utils/isEvaluationModel.d.ts.map +1 -0
- package/dist/esm/utils/isEvaluationModel.js +11 -0
- package/dist/esm/utils/isEvaluationModel.js.map +1 -0
- package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js +96 -0
- package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/src/__generated__/default_templates/index.d.ts +1 -0
- package/dist/src/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/src/__generated__/default_templates/index.js +3 -1
- package/dist/src/__generated__/default_templates/index.js.map +1 -1
- package/dist/src/llm/ClassificationEvaluator.d.ts +2 -1
- package/dist/src/llm/ClassificationEvaluator.d.ts.map +1 -1
- package/dist/src/llm/ClassificationEvaluator.js.map +1 -1
- package/dist/src/llm/createCompletenessEvaluator.d.ts +46 -0
- package/dist/src/llm/createCompletenessEvaluator.d.ts.map +1 -0
- package/dist/src/llm/createCompletenessEvaluator.js +49 -0
- package/dist/src/llm/createCompletenessEvaluator.js.map +1 -0
- package/dist/src/llm/createDocumentRelevanceEvaluator.d.ts +16 -11
- package/dist/src/llm/createDocumentRelevanceEvaluator.d.ts.map +1 -1
- package/dist/src/llm/createDocumentRelevanceEvaluator.js +7 -11
- package/dist/src/llm/createDocumentRelevanceEvaluator.js.map +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +4 -5
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js +4 -5
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js.map +1 -1
- package/dist/src/llm/generateClassification.d.ts +6 -0
- package/dist/src/llm/generateClassification.d.ts.map +1 -1
- package/dist/src/llm/generateClassification.js +65 -0
- package/dist/src/llm/generateClassification.js.map +1 -1
- package/dist/src/llm/index.d.ts +1 -0
- package/dist/src/llm/index.d.ts.map +1 -1
- package/dist/src/llm/index.js +1 -0
- package/dist/src/llm/index.js.map +1 -1
- package/dist/src/types/evals.d.ts +14 -2
- package/dist/src/types/evals.d.ts.map +1 -1
- package/dist/src/utils/bindEvaluator.d.ts +4 -5
- package/dist/src/utils/bindEvaluator.d.ts.map +1 -1
- package/dist/src/utils/bindEvaluator.js +4 -5
- package/dist/src/utils/bindEvaluator.js.map +1 -1
- package/dist/src/utils/isEvaluationModel.d.ts +13 -0
- package/dist/src/utils/isEvaluationModel.d.ts.map +1 -0
- package/dist/src/utils/isEvaluationModel.js +14 -0
- package/dist/src/utils/isEvaluationModel.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +13 -12
- package/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.ts +95 -0
- package/src/__generated__/default_templates/index.ts +1 -0
- package/src/llm/ClassificationEvaluator.ts +2 -1
- package/src/llm/createCompletenessEvaluator.ts +74 -0
- package/src/llm/createDocumentRelevanceEvaluator.ts +16 -11
- package/src/llm/createRetrievalRelevanceEvaluator.ts +4 -5
- package/src/llm/generateClassification.ts +78 -1
- package/src/llm/index.ts +1 -0
- package/src/types/evals.ts +15 -3
- package/src/utils/bindEvaluator.ts +4 -5
- package/src/utils/isEvaluationModel.ts +23 -0
|
@@ -3,6 +3,12 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
3
3
|
/**
|
|
4
4
|
* Creates a document relevance evaluator function.
|
|
5
5
|
*
|
|
6
|
+
* @deprecated Use {@link createRetrievalRelevanceEvaluator} instead. Rename the
|
|
7
|
+
* `documentText` input field to `context`, and update code that checks for the
|
|
8
|
+
* `unrelated` label to check for `irrelevant`. Pass one document as `context` to
|
|
9
|
+
* preserve per-document evaluation. This function will be removed in the next major
|
|
10
|
+
* release.
|
|
11
|
+
*
|
|
6
12
|
* This function returns an evaluator that determines whether a given document text
|
|
7
13
|
* is relevant to a provided input question. The evaluator uses a classification model
|
|
8
14
|
* and a prompt template to make its determination.
|
|
@@ -13,18 +19,8 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
13
19
|
* @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCE_TEMPLATE).
|
|
14
20
|
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
15
21
|
*
|
|
16
|
-
* @returns An evaluator function that takes a {@link
|
|
22
|
+
* @returns An evaluator function that takes a {@link DocumentRelevanceEvaluationRecord} and returns a classification result
|
|
17
23
|
* indicating whether the document is relevant to the input question.
|
|
18
|
-
*
|
|
19
|
-
* @example
|
|
20
|
-
* ```ts
|
|
21
|
-
* const evaluator = createDocumentRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
22
|
-
* const result = await evaluator.evaluate({
|
|
23
|
-
* input: "What is the capital of France?",
|
|
24
|
-
* documentText: "Paris is the capital and most populous city of France.",
|
|
25
|
-
* });
|
|
26
|
-
* console.log(result.label); // "relevant" or "unrelated"
|
|
27
|
-
* ```
|
|
28
24
|
*/
|
|
29
25
|
export function createDocumentRelevanceEvaluator(args) {
|
|
30
26
|
const { choices = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createDocumentRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kDAAkD,EAAE,MAAM,oCAAoC,CAAC;AAGxG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"createDocumentRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kDAAkD,EAAE,MAAM,oCAAoC,CAAC;AAGxG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAiChF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,gCAAgC,CAI9C,IAAgD;IAEhD,MAAM,EACJ,OAAO,GAAG,kDAAkD,CAAC,OAAO,EACpE,cAAc,GAAG,kDAAkD,CAAC,QAAQ,EAC5E,qBAAqB,GAAG,kDAAkD,CAAC,qBAAqB,EAChG,IAAI,GAAG,kDAAkD,CAAC,IAAI,EAC9D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -26,11 +26,10 @@ export interface RetrievalRelevanceEvaluationRecord {
|
|
|
26
26
|
*
|
|
27
27
|
* This function returns an evaluator that determines whether the external
|
|
28
28
|
* information retrieved during a step is relevant to the request it was
|
|
29
|
-
* serving.
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
* search, or content embedded in an LLM turn.
|
|
29
|
+
* serving. It is source-agnostic and scores the retrieved information as a
|
|
30
|
+
* whole, whether it came from a vector search, a tool or MCP call, a web search,
|
|
31
|
+
* or content embedded in an LLM turn. Pass one document as `context` to evaluate
|
|
32
|
+
* documents individually.
|
|
34
33
|
*
|
|
35
34
|
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
36
35
|
* @param args.model - The model to use for classification.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -5,11 +5,10 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
5
5
|
*
|
|
6
6
|
* This function returns an evaluator that determines whether the external
|
|
7
7
|
* information retrieved during a step is relevant to the request it was
|
|
8
|
-
* serving.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* search, or content embedded in an LLM turn.
|
|
8
|
+
* serving. It is source-agnostic and scores the retrieved information as a
|
|
9
|
+
* whole, whether it came from a vector search, a tool or MCP call, a web search,
|
|
10
|
+
* or content embedded in an LLM turn. Pass one document as `context` to evaluate
|
|
11
|
+
* documents individually.
|
|
13
12
|
*
|
|
14
13
|
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
15
14
|
* @param args.model - The model to use for classification.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,UAAU,iCAAiC,CAI/C,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,mDAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,mDAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,mDAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,mDAAmD,CAAC,IAAI,EAC/D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -17,6 +17,12 @@ export type ClassifyArgs = WithLLM & WithTelemetry & WithPrompt & {
|
|
|
17
17
|
};
|
|
18
18
|
/**
|
|
19
19
|
* A function that leverages an llm to perform a classification
|
|
20
|
+
*
|
|
21
|
+
* When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
|
|
22
|
+
* classification is routed through `experimental_evaluate` as a single
|
|
23
|
+
* choice question. Evaluation models cannot generate text, so no explanation
|
|
24
|
+
* is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
|
|
25
|
+
* are ignored on that path: `experimental_evaluate` does not emit spans yet.
|
|
20
26
|
*/
|
|
21
27
|
export declare function generateClassification(args: ClassifyArgs): Promise<ClassificationResult>;
|
|
22
28
|
//# sourceMappingURL=generateClassification.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAMnD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAgC/B"}
|
|
@@ -1,11 +1,21 @@
|
|
|
1
|
-
import { generateObject } from "ai";
|
|
1
|
+
import { experimental_evaluate, generateObject } from "ai";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { getTelemetryIntegrations, tracer } from "../telemetry/index.js";
|
|
4
|
+
import { isEvaluationModel, } from "../utils/isEvaluationModel.js";
|
|
4
5
|
/**
|
|
5
6
|
* A function that leverages an llm to perform a classification
|
|
7
|
+
*
|
|
8
|
+
* When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
|
|
9
|
+
* classification is routed through `experimental_evaluate` as a single
|
|
10
|
+
* choice question. Evaluation models cannot generate text, so no explanation
|
|
11
|
+
* is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
|
|
12
|
+
* are ignored on that path: `experimental_evaluate` does not emit spans yet.
|
|
6
13
|
*/
|
|
7
14
|
export async function generateClassification(args) {
|
|
8
15
|
const { labels, model, schemaName, schemaDescription, telemetry, ...prompt } = args;
|
|
16
|
+
if (isEvaluationModel(model)) {
|
|
17
|
+
return evaluateClassification({ model, labels, prompt });
|
|
18
|
+
}
|
|
9
19
|
const telemetryOptions = {
|
|
10
20
|
isEnabled: telemetry?.isEnabled ?? true,
|
|
11
21
|
functionId: "generateClassification",
|
|
@@ -30,4 +40,59 @@ export async function generateClassification(args) {
|
|
|
30
40
|
explanation: result.object.explanation,
|
|
31
41
|
};
|
|
32
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Classify using an AI SDK evaluation model. The rendered prompt becomes the
|
|
45
|
+
* shared evaluation state and the labels become the options of one choice
|
|
46
|
+
* question. The model's label probabilities and resolved model id are
|
|
47
|
+
* returned as metadata.
|
|
48
|
+
*/
|
|
49
|
+
async function evaluateClassification(args) {
|
|
50
|
+
const { model, labels, prompt } = args;
|
|
51
|
+
const result = await experimental_evaluate({
|
|
52
|
+
model,
|
|
53
|
+
state: toEvaluationState(prompt),
|
|
54
|
+
questions: {
|
|
55
|
+
label: {
|
|
56
|
+
type: "choice",
|
|
57
|
+
instructions: "Read the prompt in the state and answer it by selecting exactly one of the choices.",
|
|
58
|
+
criteria: Object.fromEntries(labels.map((label) => [label, null])),
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
});
|
|
62
|
+
const answer = result.answers.label;
|
|
63
|
+
return {
|
|
64
|
+
label: answer.choice,
|
|
65
|
+
metadata: {
|
|
66
|
+
probabilities: answer.probabilities,
|
|
67
|
+
modelId: result.response.modelId,
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Convert an AI SDK prompt into JSON-compatible evaluation state.
|
|
73
|
+
* A bare text prompt is passed through as a string; anything else
|
|
74
|
+
* (instructions, system messages, message arrays) is serialized to JSON.
|
|
75
|
+
*/
|
|
76
|
+
function toEvaluationState(prompt) {
|
|
77
|
+
const { instructions, system, messages, prompt: promptContent } = prompt;
|
|
78
|
+
if (typeof promptContent === "string" &&
|
|
79
|
+
instructions === undefined &&
|
|
80
|
+
system === undefined &&
|
|
81
|
+
messages === undefined) {
|
|
82
|
+
return promptContent;
|
|
83
|
+
}
|
|
84
|
+
const state = {};
|
|
85
|
+
if (instructions !== undefined)
|
|
86
|
+
state.instructions = instructions;
|
|
87
|
+
if (system !== undefined)
|
|
88
|
+
state.system = system;
|
|
89
|
+
if (promptContent !== undefined)
|
|
90
|
+
state.prompt = promptContent;
|
|
91
|
+
if (messages !== undefined)
|
|
92
|
+
state.messages = messages;
|
|
93
|
+
// Round-trip through JSON so the state only contains JSON values
|
|
94
|
+
// (strips `undefined`, turns Date/URL into strings). Binary file parts
|
|
95
|
+
// are not supported by evaluation models and are not stripped here.
|
|
96
|
+
return JSON.parse(JSON.stringify(state));
|
|
97
|
+
}
|
|
33
98
|
//# sourceMappingURL=generateClassification.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AAC3D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,wBAAwB,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAIhE,OAAO,EAEL,iBAAiB,GAClB,MAAM,4BAA4B,CAAC;AAkBpC;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAC1E,IAAI,CAAC;IAEP,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,sBAAsB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,gBAAgB,GAAG;QACvB,SAAS,EAAE,SAAS,EAAE,SAAS,IAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,YAAY,EAAE,wBAAwB,CAAC,SAAS,EAAE,MAAM,IAAI,MAAM,CAAC;KACpE,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,UAAU;QACV,iBAAiB;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC;QACF,SAAS,EAAE,gBAAgB;QAC3B,sEAAsE;QACtE,qEAAqE;QACrE,qBAAqB,EAAE,IAAI;QAC3B,GAAG,MAAM;KACV,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC;AAID;;;;;GAKG;AACH,KAAK,UAAU,sBAAsB,CAAC,IAIrC;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC;QACzC,KAAK;QACL,KAAK,EAAE,iBAAiB,CAAC,MAAM,CAAC;QAChC,SAAS,EAAE;YACT,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,YAAY,EACV,qFAAqF;gBACvF,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;KACF,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;IACpC,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,MAAkB;IAC3C,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,CAAC;IACzE,IACE,OAAO,aAAa,KAAK,QAAQ;QACjC,YAAY,KAAK,SAAS;QAC1B,MAAM,KAAK,SAAS;QACpB,QAAQ,KAAK,SAAS,EACtB,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,IAAI,YAAY,KAAK,SAAS;QAAE,KAAK,CAAC,YAAY,GAAG,YAAY,CAAC;IAClE,IAAI,MAAM,KAAK,SAAS;QAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IAChD,IAAI,aAAa,KAAK,SAAS;QAAE,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC;IAC9D,IAAI,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACtD,iEAAiE;IACjE,uEAAuE;IACvE,oEAAoE;IACpE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAoB,CAAC;AAC9D,CAAC"}
|
package/dist/esm/llm/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export * from "./ClassificationEvaluator.js";
|
|
2
2
|
export * from "./createClassificationEvaluator.js";
|
|
3
3
|
export * from "./createClassifierFn.js";
|
|
4
|
+
export * from "./createCompletenessEvaluator.js";
|
|
4
5
|
export * from "./createConcisenessEvaluator.js";
|
|
5
6
|
export * from "./createCorrectnessEvaluator.js";
|
|
6
7
|
export * from "./createDocumentRelevanceEvaluator.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
package/dist/esm/llm/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export * from "./ClassificationEvaluator.js";
|
|
2
2
|
export * from "./createClassificationEvaluator.js";
|
|
3
3
|
export * from "./createClassifierFn.js";
|
|
4
|
+
export * from "./createCompletenessEvaluator.js";
|
|
4
5
|
export * from "./createConcisenessEvaluator.js";
|
|
5
6
|
export * from "./createCorrectnessEvaluator.js";
|
|
6
7
|
export * from "./createDocumentRelevanceEvaluator.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|