@arizeai/phoenix-evals 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js +62 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/esm/__generated__/default_templates/index.d.ts +1 -0
- package/dist/esm/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/esm/__generated__/default_templates/index.js +1 -0
- package/dist/esm/__generated__/default_templates/index.js.map +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +55 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +43 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js.map +1 -0
- package/dist/esm/llm/index.d.ts +1 -0
- package/dist/esm/llm/index.d.ts.map +1 -1
- package/dist/esm/llm/index.js +1 -0
- package/dist/esm/llm/index.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js +65 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/src/__generated__/default_templates/index.d.ts +1 -0
- package/dist/src/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/src/__generated__/default_templates/index.js +3 -1
- package/dist/src/__generated__/default_templates/index.js.map +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +55 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js +54 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js.map +1 -0
- package/dist/src/llm/index.d.ts +1 -0
- package/dist/src/llm/index.d.ts.map +1 -1
- package/dist/src/llm/index.js +1 -0
- package/dist/src/llm/index.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +10 -10
- package/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts +64 -0
- package/src/__generated__/default_templates/index.ts +1 -0
- package/src/llm/createRetrievalRelevanceEvaluator.ts +85 -0
- package/src/llm/index.ts +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,mDAAmD,EAAE,6BA2DjE,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// This file is generated. Do not edit by hand.
|
|
2
|
+
export const RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = {
|
|
3
|
+
name: "retrieval_relevance",
|
|
4
|
+
description: "Determine whether the external information retrieved during a step is relevant to the request it was serving, regardless of the source (vector search, tool call, MCP server, or web search).",
|
|
5
|
+
optimizationDirection: "MAXIMIZE",
|
|
6
|
+
template: [
|
|
7
|
+
{
|
|
8
|
+
role: "user",
|
|
9
|
+
content: `
|
|
10
|
+
You are an expert evaluator judging whether externally retrieved information is RELEVANT to the request it was meant to serve.
|
|
11
|
+
|
|
12
|
+
The request is everything that defines what the retrieval needed to satisfy: the user's question or task, and any earlier turns or context needed to interpret it. The retrieved information is everything that came back from the retrieval step — a vector/semantic search, a tool or function call, an MCP server, a web search, or a database query — regardless of how it is formatted (plain text, structured data, or a rendered result). Treat all sources the same way: you are judging the retrieved information as a whole against the request, not judging the source's general trustworthiness.
|
|
13
|
+
|
|
14
|
+
<rubric>
|
|
15
|
+
|
|
16
|
+
RELEVANT - The retrieved information contains content that materially helps address the request. This includes when it:
|
|
17
|
+
|
|
18
|
+
- Directly answers the request, or
|
|
19
|
+
- Provides facts, data, or context needed to answer the request, or
|
|
20
|
+
- Is on-topic and useful even if it only partially covers the request, or is mixed with some unrelated material, or
|
|
21
|
+
- Is accurate as of a different time, superseded, or later contradicted, as long as it was genuinely about the entity/topic the request concerns.
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
IRRELEVANT - The retrieved information does NOT contain content that helps address the request. This includes when it:
|
|
25
|
+
|
|
26
|
+
- Is about a different topic, entity, or time period than the request asks about, or
|
|
27
|
+
- Is only superficially or tangentially related — e.g. shares a keyword but not the substance of the request — and does not help answer it, or
|
|
28
|
+
- Is empty, an error, a timeout, or a "no results found" response, or
|
|
29
|
+
|
|
30
|
+
</rubric>
|
|
31
|
+
|
|
32
|
+
Apply these rules when deciding:
|
|
33
|
+
|
|
34
|
+
1. Partial relevance is enough. If any meaningful part of the retrieved information bears on the request, the whole retrieval step is RELEVANT — even if most of the content is unrelated or unused.
|
|
35
|
+
2. Relevance is not correctness. Judge only whether the information bears on the request's topic, not whether it is accurate, current, or complete. Wrong, outdated, or later-contradicted information is still RELEVANT if it was genuinely about the right subject.
|
|
36
|
+
3. A failed retrieval is not relevant. An error, timeout, or empty result provides no usable content, so it is IRRELEVANT — regardless of whether the underlying lookup, if it had succeeded, would have been on-topic.
|
|
37
|
+
4. Judge the retrieved information as a whole. Do not require every sentence or every retrieved item to be relevant individually; ask whether the set collectively contains something useful.
|
|
38
|
+
5. Superficial topical overlap is not enough. Sharing an entity name, keyword, or category with the request does not make information relevant if it doesn't actually address what the request needed.
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
<data>
|
|
42
|
+
|
|
43
|
+
<request>
|
|
44
|
+
{{input}}
|
|
45
|
+
</request>
|
|
46
|
+
|
|
47
|
+
<retrieved_information>
|
|
48
|
+
{{context}}
|
|
49
|
+
</retrieved_information>
|
|
50
|
+
|
|
51
|
+
</data>
|
|
52
|
+
|
|
53
|
+
Work through the retrieved information piece by piece, checking each part against the request, before you decide. Is the retrieved information relevant or irrelevant?
|
|
54
|
+
`,
|
|
55
|
+
},
|
|
56
|
+
],
|
|
57
|
+
choices: {
|
|
58
|
+
"relevant": 1,
|
|
59
|
+
"irrelevant": 0
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
//# sourceMappingURL=RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAI/C,MAAM,CAAC,MAAM,mDAAmD,GAAkC;IAChG,IAAI,EAAE,qBAAqB;IAC3B,WAAW,EAAE,+LAA+L;IAC5M,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Cd;SACI;KACF;IACD,OAAO,EAAE;QACT,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;KAChB;CACA,CAAC"}
|
|
@@ -4,6 +4,7 @@ export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_R
|
|
|
4
4
|
export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
5
5
|
export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
6
6
|
export { REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG } from "./REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
7
|
+
export { RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
7
8
|
export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
8
9
|
export { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
9
10
|
export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,mDAAmD,EAAE,MAAM,uDAAuD,CAAC;AAC5H,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
@@ -5,6 +5,7 @@ export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_R
|
|
|
5
5
|
export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
6
6
|
export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
7
7
|
export { REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG } from "./REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
8
|
+
export { RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
8
9
|
export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
9
10
|
export { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
10
11
|
export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,mDAAmD,EAAE,MAAM,uDAAuD,CAAC;AAC5H,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { CreateClassificationEvaluatorArgs } from "../types/evals.js";
|
|
2
|
+
import type { ClassificationEvaluator } from "./ClassificationEvaluator.js";
|
|
3
|
+
export interface RetrievalRelevanceEvaluatorArgs<RecordType extends Record<string, unknown> = RetrievalRelevanceEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
|
|
4
|
+
optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
|
|
5
|
+
name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
|
|
6
|
+
choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
|
|
7
|
+
promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
|
|
8
|
+
}
|
|
9
|
+
/** A request and the external information retrieved to serve it. */
|
|
10
|
+
export interface RetrievalRelevanceEvaluationRecord {
|
|
11
|
+
/**
|
|
12
|
+
* The request the retrieval was serving. Prefer the user's request (e.g. the
|
|
13
|
+
* trace root's input) over a reformulated tool argument or generated query.
|
|
14
|
+
*/
|
|
15
|
+
input: string;
|
|
16
|
+
/**
|
|
17
|
+
* The external information retrieved during the step, with all returned items
|
|
18
|
+
* joined together. Source-agnostic: vector search, tool/MCP call, web search,
|
|
19
|
+
* or content embedded in an LLM turn.
|
|
20
|
+
*/
|
|
21
|
+
context: string;
|
|
22
|
+
[key: string]: unknown;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Creates a retrieval relevance evaluator function.
|
|
26
|
+
*
|
|
27
|
+
* This function returns an evaluator that determines whether the external
|
|
28
|
+
* information retrieved during a step is relevant to the request it was
|
|
29
|
+
* serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
|
|
30
|
+
* single document against a question, this evaluator is source-agnostic and
|
|
31
|
+
* scores the retrieved information as a whole (holistically, per retrieval
|
|
32
|
+
* step), whether it came from a vector search, a tool or MCP call, a web
|
|
33
|
+
* search, or content embedded in an LLM turn.
|
|
34
|
+
*
|
|
35
|
+
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
36
|
+
* @param args.model - The model to use for classification.
|
|
37
|
+
* @param args.choices - The possible classification choices (defaults to relevant/irrelevant).
|
|
38
|
+
* @param args.promptTemplate - The prompt template to use (defaults to RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
39
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
40
|
+
*
|
|
41
|
+
* @returns An evaluator function that takes a {@link RetrievalRelevanceEvaluationRecord} and returns a classification result
|
|
42
|
+
* indicating whether the retrieved information is relevant or irrelevant to the request.
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* ```ts
|
|
46
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
47
|
+
* const result = await evaluator.evaluate({
|
|
48
|
+
* input: "What is the capital of France?",
|
|
49
|
+
* context: "Paris is the capital and largest city of France.",
|
|
50
|
+
* });
|
|
51
|
+
* console.log(result.label); // "relevant"
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
export declare function createRetrievalRelevanceEvaluator<RecordType extends Record<string, unknown> = RetrievalRelevanceEvaluationRecord>(args: RetrievalRelevanceEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
|
|
55
|
+
//# sourceMappingURL=createRetrievalRelevanceEvaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
|
|
2
|
+
import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a retrieval relevance evaluator function.
|
|
5
|
+
*
|
|
6
|
+
* This function returns an evaluator that determines whether the external
|
|
7
|
+
* information retrieved during a step is relevant to the request it was
|
|
8
|
+
* serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
|
|
9
|
+
* single document against a question, this evaluator is source-agnostic and
|
|
10
|
+
* scores the retrieved information as a whole (holistically, per retrieval
|
|
11
|
+
* step), whether it came from a vector search, a tool or MCP call, a web
|
|
12
|
+
* search, or content embedded in an LLM turn.
|
|
13
|
+
*
|
|
14
|
+
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
15
|
+
* @param args.model - The model to use for classification.
|
|
16
|
+
* @param args.choices - The possible classification choices (defaults to relevant/irrelevant).
|
|
17
|
+
* @param args.promptTemplate - The prompt template to use (defaults to RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
18
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
19
|
+
*
|
|
20
|
+
* @returns An evaluator function that takes a {@link RetrievalRelevanceEvaluationRecord} and returns a classification result
|
|
21
|
+
* indicating whether the retrieved information is relevant or irrelevant to the request.
|
|
22
|
+
*
|
|
23
|
+
* @example
|
|
24
|
+
* ```ts
|
|
25
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
26
|
+
* const result = await evaluator.evaluate({
|
|
27
|
+
* input: "What is the capital of France?",
|
|
28
|
+
* context: "Paris is the capital and largest city of France.",
|
|
29
|
+
* });
|
|
30
|
+
* console.log(result.label); // "relevant"
|
|
31
|
+
* ```
|
|
32
|
+
*/
|
|
33
|
+
export function createRetrievalRelevanceEvaluator(args) {
|
|
34
|
+
const { choices = RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
|
|
35
|
+
return createClassificationEvaluator({
|
|
36
|
+
...rest,
|
|
37
|
+
promptTemplate,
|
|
38
|
+
choices,
|
|
39
|
+
optimizationDirection,
|
|
40
|
+
name,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=createRetrievalRelevanceEvaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,UAAU,iCAAiC,CAI/C,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,mDAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,mDAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,mDAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,mDAAmD,CAAC,IAAI,EAC/D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
package/dist/esm/llm/index.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export * from "./createDocumentRelevanceEvaluator.js";
|
|
|
7
7
|
export * from "./createFaithfulnessEvaluator.js";
|
|
8
8
|
export * from "./createHallucinationEvaluator.js";
|
|
9
9
|
export * from "./createRefusalEvaluator.js";
|
|
10
|
+
export * from "./createRetrievalRelevanceEvaluator.js";
|
|
10
11
|
export * from "./createToolInvocationEvaluator.js";
|
|
11
12
|
export * from "./createToolResponseHandlingEvaluator.js";
|
|
12
13
|
export * from "./createToolSelectionEvaluator.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
package/dist/esm/llm/index.js
CHANGED
|
@@ -7,6 +7,7 @@ export * from "./createDocumentRelevanceEvaluator.js";
|
|
|
7
7
|
export * from "./createFaithfulnessEvaluator.js";
|
|
8
8
|
export * from "./createHallucinationEvaluator.js";
|
|
9
9
|
export * from "./createRefusalEvaluator.js";
|
|
10
|
+
export * from "./createRetrievalRelevanceEvaluator.js";
|
|
10
11
|
export * from "./createToolInvocationEvaluator.js";
|
|
11
12
|
export * from "./createToolResponseHandlingEvaluator.js";
|
|
12
13
|
export * from "./createToolSelectionEvaluator.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|