@arizeai/phoenix-evals 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -1
- package/dist/esm/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js +48 -17
- package/dist/esm/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -1
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js +62 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/esm/__generated__/default_templates/index.d.ts +1 -0
- package/dist/esm/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/esm/__generated__/default_templates/index.js +1 -0
- package/dist/esm/__generated__/default_templates/index.js.map +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.d.ts +36 -19
- package/dist/esm/llm/createHallucinationEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.js +24 -18
- package/dist/esm/llm/createHallucinationEvaluator.js.map +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +55 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +43 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js.map +1 -0
- package/dist/esm/llm/index.d.ts +1 -0
- package/dist/esm/llm/index.d.ts.map +1 -1
- package/dist/esm/llm/index.js +2 -1
- package/dist/esm/llm/index.js.map +1 -1
- package/dist/esm/template/getTemplateVariables.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -1
- package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js +48 -17
- package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -1
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js +65 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/src/__generated__/default_templates/index.d.ts +1 -0
- package/dist/src/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/src/__generated__/default_templates/index.js +3 -1
- package/dist/src/__generated__/default_templates/index.js.map +1 -1
- package/dist/src/llm/createHallucinationEvaluator.d.ts +36 -19
- package/dist/src/llm/createHallucinationEvaluator.d.ts.map +1 -1
- package/dist/src/llm/createHallucinationEvaluator.js +24 -18
- package/dist/src/llm/createHallucinationEvaluator.js.map +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +55 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js +54 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js.map +1 -0
- package/dist/src/llm/index.d.ts +1 -0
- package/dist/src/llm/index.d.ts.map +1 -1
- package/dist/src/llm/index.js +2 -1
- package/dist/src/llm/index.js.map +1 -1
- package/dist/src/template/getTemplateVariables.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +11 -11
- package/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts +48 -17
- package/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts +64 -0
- package/src/__generated__/default_templates/index.ts +1 -0
- package/src/llm/createHallucinationEvaluator.ts +36 -28
- package/src/llm/createRetrievalRelevanceEvaluator.ts +85 -0
- package/src/llm/index.ts +2 -1
- package/src/template/getTemplateVariables.ts +4 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,6CAA6C,EAAE,
|
|
1
|
+
{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,6CAA6C,EAAE,6BA2E3D,CAAC"}
|
package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js
CHANGED
|
@@ -4,47 +4,78 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
4
4
|
exports.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
|
|
5
5
|
exports.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG = {
|
|
6
6
|
name: "hallucination",
|
|
7
|
-
description: "
|
|
7
|
+
description: "Detect whether an assistant's response contains hallucinations — claims not supported by the input it came from (prior turns, tool calls and results, and any content read, retrieved, or provided).",
|
|
8
8
|
optimizationDirection: "MINIMIZE",
|
|
9
9
|
template: [
|
|
10
10
|
{
|
|
11
11
|
role: "user",
|
|
12
12
|
content: `
|
|
13
|
-
|
|
13
|
+
You are an expert evaluator labeling whether an assistant's response is grounded in the input it came from.
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
The input is the complete record of what the assistant had available before it responded: the user's request, any earlier turns, any tool calls and the results they returned, and any content that was read, retrieved, or provided. It may appear as plain text, as structured data, or as a rendered transcript. Whatever form it takes, treat it as the full record of what the assistant knew and did, and as the only source of truth for claims about this situation. You are judging whether the response is GROUNDED in that record, not whether it is factually true about the world. The output is the response you are judging.
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
<rubric>
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
HALLUCINATED - The response contains at least one claim that the input does not support. This includes:
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
- Contradiction: the claim conflicts with something in the input.
|
|
22
|
+
- Invented specifics: names, numbers, prices, dates, quotes, identifiers, entities, or events that do not appear in the input and do not follow from it.
|
|
23
|
+
- Fabricated tool results: the response reports what a tool returned, but the input shows no such result — or shows a different one.
|
|
24
|
+
- Unsupported findings: the response presents findings, file contents, search results, sources, or a review of material that the input never shows being read, searched, or returned.
|
|
25
|
+
- Fabricated completed actions: the response states that an action has already been taken when the input shows no record of it happening.
|
|
26
|
+
- Misattribution: the response attributes a claim, source, or action to the input that never appears there.
|
|
22
27
|
|
|
23
|
-
|
|
28
|
+
GROUNDED - Every claim in the response is supported. The response:
|
|
29
|
+
|
|
30
|
+
- Restates or summarizes what is in the input, accurately.
|
|
31
|
+
- Draws conclusions that follow necessarily from the input.
|
|
32
|
+
- Uses only uncontroversial, widely-known general knowledge — common facts, definitions, arithmetic, units of time or measure — that does not contradict the input. This is the one kind of claim that need not appear in the input (see rule 4).
|
|
33
|
+
- Declines to answer, reports a failure, expresses uncertainty, or asks a clarifying question without asserting unsupported specifics.
|
|
34
|
+
|
|
35
|
+
</rubric>
|
|
36
|
+
|
|
37
|
+
Apply these rules when deciding:
|
|
38
|
+
|
|
39
|
+
1. Absence of evidence is enough. You do not need to find a contradiction. If the response asserts situation-specific facts that simply do not appear anywhere in the input, that is a hallucination — however plausible, fluent, or well-organized the response is. A confident report is not evidence that the work behind it was done.
|
|
40
|
+
|
|
41
|
+
2. Not everything in the input carries the same weight. User messages, tool results, and content that was read or provided are authoritative evidence. The assistant's own earlier statements are part of the record but are not independent evidence for a claim.
|
|
42
|
+
|
|
43
|
+
3. A failed tool is not a source. An error, timeout, empty result, or "not found" is evidence that the lookup failed. It is not evidence about the thing being looked up. A response that supplies the missing value anyway is hallucinating.
|
|
44
|
+
|
|
45
|
+
4. Apply the absence-of-evidence rule only to situation-specific claims. First distinguish claims about this particular situation — its entities, values, dates, events, sources, and what was returned or done — from ordinary general knowledge. Situation-specific claims must restate the input or follow from it necessarily; they are not supported if deriving them requires adding a typical value, default, cause, external lookup, or plausible completion. Ordinary general knowledge — definitions, arithmetic, common units and conventions, and widely known facts — is outside the grounding check and cannot by itself make a response hallucinated, as long as it does not contradict the input.
|
|
46
|
+
|
|
47
|
+
5. One unsupported claim is enough. If any claim in the response fails these tests, the response is hallucinated. Quote the specific claim in your explanation.
|
|
48
|
+
|
|
49
|
+
Out of scope — do not label a response hallucinated for any of these:
|
|
50
|
+
|
|
51
|
+
- Being unhelpful, incomplete, poorly written, off-topic, or badly formatted.
|
|
52
|
+
- Failing to do everything the user asked. Whether the assistant covered the full request is separate from whether what it said is supported.
|
|
53
|
+
- Stating an intention to do something in the future. "I'll open a ticket" is a plan, not a claim about what happened. Only claims that an action is already complete are checkable here.
|
|
54
|
+
- Content the input shows was shortened or omitted. Do not treat the absence of a claim from an explicitly elided region as evidence that the claim was fabricated. If the omitted region could plausibly contain the missing support, default to grounded. Otherwise, judge the claim against the visible input normally.
|
|
55
|
+
|
|
56
|
+
When you are genuinely unsure whether a claim is supported, label the response grounded.
|
|
24
57
|
|
|
25
58
|
<data>
|
|
26
59
|
|
|
27
|
-
<
|
|
60
|
+
<input>
|
|
28
61
|
{{input}}
|
|
29
|
-
</
|
|
30
|
-
|
|
31
|
-
<context>
|
|
32
|
-
{{context}}
|
|
33
|
-
</context>
|
|
62
|
+
</input>
|
|
34
63
|
|
|
35
|
-
<
|
|
64
|
+
<output>
|
|
36
65
|
{{output}}
|
|
37
|
-
</
|
|
66
|
+
</output>
|
|
38
67
|
|
|
39
68
|
</data>
|
|
40
69
|
|
|
41
|
-
|
|
70
|
+
Work through the claims in the output one at a time, checking each against the input, before you decide.
|
|
71
|
+
|
|
72
|
+
Is the output above grounded or hallucinated?
|
|
42
73
|
`,
|
|
43
74
|
},
|
|
44
75
|
],
|
|
45
76
|
choices: {
|
|
46
77
|
"hallucinated": 1,
|
|
47
|
-
"
|
|
78
|
+
"grounded": 0
|
|
48
79
|
},
|
|
49
80
|
};
|
|
50
81
|
//# sourceMappingURL=HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,6CAA6C,GAAkC;IAC1F,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,
|
|
1
|
+
{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,6CAA6C,GAAkC;IAC1F,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,sMAAsM;IACnN,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Dd;SACI;KACF;IACD,OAAO,EAAE;QACT,cAAc,EAAE,CAAC;QACjB,UAAU,EAAE,CAAC;KACd;CACA,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,mDAAmD,EAAE,6BA2DjE,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// This file is generated. Do not edit by hand.
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
|
|
5
|
+
exports.RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = {
|
|
6
|
+
name: "retrieval_relevance",
|
|
7
|
+
description: "Determine whether the external information retrieved during a step is relevant to the request it was serving, regardless of the source (vector search, tool call, MCP server, or web search).",
|
|
8
|
+
optimizationDirection: "MAXIMIZE",
|
|
9
|
+
template: [
|
|
10
|
+
{
|
|
11
|
+
role: "user",
|
|
12
|
+
content: `
|
|
13
|
+
You are an expert evaluator judging whether externally retrieved information is RELEVANT to the request it was meant to serve.
|
|
14
|
+
|
|
15
|
+
The request is everything that defines what the retrieval needed to satisfy: the user's question or task, and any earlier turns or context needed to interpret it. The retrieved information is everything that came back from the retrieval step — a vector/semantic search, a tool or function call, an MCP server, a web search, or a database query — regardless of how it is formatted (plain text, structured data, or a rendered result). Treat all sources the same way: you are judging the retrieved information as a whole against the request, not judging the source's general trustworthiness.
|
|
16
|
+
|
|
17
|
+
<rubric>
|
|
18
|
+
|
|
19
|
+
RELEVANT - The retrieved information contains content that materially helps address the request. This includes when it:
|
|
20
|
+
|
|
21
|
+
- Directly answers the request, or
|
|
22
|
+
- Provides facts, data, or context needed to answer the request, or
|
|
23
|
+
- Is on-topic and useful even if it only partially covers the request, or is mixed with some unrelated material, or
|
|
24
|
+
- Is accurate as of a different time, superseded, or later contradicted, as long as it was genuinely about the entity/topic the request concerns.
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
IRRELEVANT - The retrieved information does NOT contain content that helps address the request. This includes when it:
|
|
28
|
+
|
|
29
|
+
- Is about a different topic, entity, or time period than the request asks about, or
|
|
30
|
+
- Is only superficially or tangentially related — e.g. shares a keyword but not the substance of the request — and does not help answer it, or
|
|
31
|
+
- Is empty, an error, a timeout, or a "no results found" response, or
|
|
32
|
+
|
|
33
|
+
</rubric>
|
|
34
|
+
|
|
35
|
+
Apply these rules when deciding:
|
|
36
|
+
|
|
37
|
+
1. Partial relevance is enough. If any meaningful part of the retrieved information bears on the request, the whole retrieval step is RELEVANT — even if most of the content is unrelated or unused.
|
|
38
|
+
2. Relevance is not correctness. Judge only whether the information bears on the request's topic, not whether it is accurate, current, or complete. Wrong, outdated, or later-contradicted information is still RELEVANT if it was genuinely about the right subject.
|
|
39
|
+
3. A failed retrieval is not relevant. An error, timeout, or empty result provides no usable content, so it is IRRELEVANT — regardless of whether the underlying lookup, if it had succeeded, would have been on-topic.
|
|
40
|
+
4. Judge the retrieved information as a whole. Do not require every sentence or every retrieved item to be relevant individually; ask whether the set collectively contains something useful.
|
|
41
|
+
5. Superficial topical overlap is not enough. Sharing an entity name, keyword, or category with the request does not make information relevant if it doesn't actually address what the request needed.
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
<data>
|
|
45
|
+
|
|
46
|
+
<request>
|
|
47
|
+
{{input}}
|
|
48
|
+
</request>
|
|
49
|
+
|
|
50
|
+
<retrieved_information>
|
|
51
|
+
{{context}}
|
|
52
|
+
</retrieved_information>
|
|
53
|
+
|
|
54
|
+
</data>
|
|
55
|
+
|
|
56
|
+
Work through the retrieved information piece by piece, checking each part against the request, before you decide. Is the retrieved information relevant or irrelevant?
|
|
57
|
+
`,
|
|
58
|
+
},
|
|
59
|
+
],
|
|
60
|
+
choices: {
|
|
61
|
+
"relevant": 1,
|
|
62
|
+
"irrelevant": 0
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
//# sourceMappingURL=RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,mDAAmD,GAAkC;IAChG,IAAI,EAAE,qBAAqB;IAC3B,WAAW,EAAE,+LAA+L;IAC5M,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Cd;SACI;KACF;IACD,OAAO,EAAE;QACT,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;KAChB;CACA,CAAC"}
|
|
@@ -4,6 +4,7 @@ export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_R
|
|
|
4
4
|
export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG";
|
|
5
5
|
export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG";
|
|
6
6
|
export { REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG } from "./REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG";
|
|
7
|
+
export { RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG";
|
|
7
8
|
export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG";
|
|
8
9
|
export { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG";
|
|
9
10
|
export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,mDAAmD,EAAE,MAAM,uDAAuD,CAAC;AAC5H,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// This file is generated. Do not edit by hand.
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.USER_FRICTION_CLASSIFICATION_EVALUATOR_CONFIG = exports.TOXICITY_CLASSIFICATION_EVALUATOR_CONFIG = exports.TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG = exports.TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG = exports.TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG = exports.REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG = exports.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG = exports.FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG = exports.DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = exports.CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG = exports.CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
|
|
4
|
+
exports.USER_FRICTION_CLASSIFICATION_EVALUATOR_CONFIG = exports.TOXICITY_CLASSIFICATION_EVALUATOR_CONFIG = exports.TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG = exports.TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG = exports.TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG = exports.RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = exports.REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG = exports.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG = exports.FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG = exports.DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = exports.CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG = exports.CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
|
|
5
5
|
var CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG");
|
|
6
6
|
Object.defineProperty(exports, "CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG", { enumerable: true, get: function () { return CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG_1.CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG; } });
|
|
7
7
|
var CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG");
|
|
@@ -14,6 +14,8 @@ var HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./HALLUCINATION_C
|
|
|
14
14
|
Object.defineProperty(exports, "HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG", { enumerable: true, get: function () { return HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG_1.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG; } });
|
|
15
15
|
var REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG");
|
|
16
16
|
Object.defineProperty(exports, "REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG", { enumerable: true, get: function () { return REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG_1.REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG; } });
|
|
17
|
+
var RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG");
|
|
18
|
+
Object.defineProperty(exports, "RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG", { enumerable: true, get: function () { return RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG_1.RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG; } });
|
|
17
19
|
var TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG");
|
|
18
20
|
Object.defineProperty(exports, "TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG", { enumerable: true, get: function () { return TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG_1.TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG; } });
|
|
19
21
|
var TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG_1 = require("./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAE/C,6GAA4G;AAAnG,0KAAA,2CAA2C,OAAA;AACpD,6GAA4G;AAAnG,0KAAA,2CAA2C,OAAA;AACpD,2HAA0H;AAAjH,wLAAA,kDAAkD,OAAA;AAC3D,+GAA8G;AAArG,4KAAA,4CAA4C,OAAA;AACrD,iHAAgH;AAAvG,8KAAA,6CAA6C,OAAA;AACtD,qGAAoG;AAA3F,kKAAA,uCAAuC,OAAA;AAChD,qHAAoH;AAA3G,kLAAA,+CAA+C,OAAA;AACxD,mIAAkI;AAAzH,gMAAA,sDAAsD,OAAA;AAC/D,mHAAkH;AAAzG,gLAAA,8CAA8C,OAAA;AACvD,uGAAsG;AAA7F,oKAAA,wCAAwC,OAAA;AACjD,iHAAgH;AAAvG,8KAAA,6CAA6C,OAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAE/C,6GAA4G;AAAnG,0KAAA,2CAA2C,OAAA;AACpD,6GAA4G;AAAnG,0KAAA,2CAA2C,OAAA;AACpD,2HAA0H;AAAjH,wLAAA,kDAAkD,OAAA;AAC3D,+GAA8G;AAArG,4KAAA,4CAA4C,OAAA;AACrD,iHAAgH;AAAvG,8KAAA,6CAA6C,OAAA;AACtD,qGAAoG;AAA3F,kKAAA,uCAAuC,OAAA;AAChD,6HAA4H;AAAnH,0LAAA,mDAAmD,OAAA;AAC5D,qHAAoH;AAA3G,kLAAA,+CAA+C,OAAA;AACxD,mIAAkI;AAAzH,gMAAA,sDAAsD,OAAA;AAC/D,mHAAkH;AAAzG,gLAAA,8CAA8C,OAAA;AACvD,uGAAsG;AAA7F,oKAAA,wCAAwC,OAAA;AACjD,iHAAgH;AAAvG,8KAAA,6CAA6C,OAAA"}
|
|
@@ -1,9 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @deprecated This evaluator is maintained for backwards compatibility.
|
|
3
|
-
* Please use createFaithfulnessEvaluator instead, which uses updated terminology:
|
|
4
|
-
* - 'faithful'/'unfaithful' labels instead of 'factual'/'hallucinated'
|
|
5
|
-
* - Maximizes score (1.0=faithful) instead of minimizing it
|
|
6
|
-
*/
|
|
7
1
|
import type { CreateClassificationEvaluatorArgs } from "../types/evals";
|
|
8
2
|
import type { ClassificationEvaluator } from "./ClassificationEvaluator";
|
|
9
3
|
export interface HallucinationEvaluatorArgs<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
|
|
@@ -12,25 +6,48 @@ export interface HallucinationEvaluatorArgs<RecordType extends Record<string, un
|
|
|
12
6
|
choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
|
|
13
7
|
promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
|
|
14
8
|
}
|
|
15
|
-
/**
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
9
|
+
/** A conversation and the assistant response to evaluate for hallucination. */
|
|
10
|
+
export interface HallucinationEvaluationRecord {
|
|
11
|
+
/**
|
|
12
|
+
* The full conversation history the assistant had access to (prior turns,
|
|
13
|
+
* tool calls, and tool results); its last message is the user turn being
|
|
14
|
+
* answered. Treated as the source of truth.
|
|
15
|
+
*/
|
|
19
16
|
input: string;
|
|
17
|
+
/**
|
|
18
|
+
* The assistant's latest response to classify.
|
|
19
|
+
*/
|
|
20
20
|
output: string;
|
|
21
|
-
|
|
22
|
-
}
|
|
21
|
+
[key: string]: unknown;
|
|
22
|
+
}
|
|
23
23
|
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* Creates a function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
|
|
24
|
+
* Creates a hallucination evaluator function.
|
|
27
25
|
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
26
|
+
* This function returns an evaluator that detects whether an assistant's latest
|
|
27
|
+
* response contains claims that are not grounded in — unsupported by, or
|
|
28
|
+
* contradicting — the conversation. Unlike the faithfulness evaluator, which
|
|
29
|
+
* grounds a response against a specific provided context (e.g. retrieved
|
|
30
|
+
* documents), this grounds it against the conversation itself.
|
|
31
31
|
*
|
|
32
32
|
* @param args - The arguments for creating the hallucination evaluator.
|
|
33
|
-
* @
|
|
33
|
+
* @param args.model - The model to use for classification.
|
|
34
|
+
* @param args.choices - The possible classification choices (defaults to hallucinated/grounded).
|
|
35
|
+
* @param args.promptTemplate - The prompt template to use (defaults to HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
36
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
37
|
+
*
|
|
38
|
+
* @returns An evaluator function that takes a {@link HallucinationEvaluationRecord} and returns a classification result
|
|
39
|
+
* indicating whether the response is grounded or hallucinated relative to the conversation.
|
|
40
|
+
*
|
|
41
|
+
* @example
|
|
42
|
+
* ```ts
|
|
43
|
+
* const evaluator = createHallucinationEvaluator({ model: openai("gpt-4o-mini") });
|
|
44
|
+
* const result = await evaluator.evaluate({
|
|
45
|
+
* input:
|
|
46
|
+
* "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
|
|
47
|
+
* output: "Electronics can be returned within 90 days.",
|
|
48
|
+
* });
|
|
49
|
+
* console.log(result.label); // "hallucinated"
|
|
50
|
+
* ```
|
|
34
51
|
*/
|
|
35
52
|
export declare function createHallucinationEvaluator<RecordType extends Record<string, unknown> = HallucinationEvaluationRecord>(args: HallucinationEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
|
|
36
53
|
//# sourceMappingURL=createHallucinationEvaluator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,0BAA0B,CACzC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,CAC1E,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,+EAA+E;AAC/E,MAAM,WAAW,6BAA6B;IAC5C;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAE1E,IAAI,EAAE,0BAA0B,CAAC,UAAU,CAAC,GAC3C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -1,10 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* @deprecated This evaluator is maintained for backwards compatibility.
|
|
4
|
-
* Please use createFaithfulnessEvaluator instead, which uses updated terminology:
|
|
5
|
-
* - 'faithful'/'unfaithful' labels instead of 'factual'/'hallucinated'
|
|
6
|
-
* - Maximizes score (1.0=faithful) instead of minimizing it
|
|
7
|
-
*/
|
|
8
2
|
var __rest = (this && this.__rest) || function (s, e) {
|
|
9
3
|
var t = {};
|
|
10
4
|
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
@@ -21,23 +15,35 @@ exports.createHallucinationEvaluator = createHallucinationEvaluator;
|
|
|
21
15
|
const default_templates_1 = require("../__generated__/default_templates");
|
|
22
16
|
const createClassificationEvaluator_1 = require("./createClassificationEvaluator");
|
|
23
17
|
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* Creates a function that evaluates whether an answer is factual or hallucinated based on a query and reference text.
|
|
18
|
+
* Creates a hallucination evaluator function.
|
|
27
19
|
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
20
|
+
* This function returns an evaluator that detects whether an assistant's latest
|
|
21
|
+
* response contains claims that are not grounded in — unsupported by, or
|
|
22
|
+
* contradicting — the conversation. Unlike the faithfulness evaluator, which
|
|
23
|
+
* grounds a response against a specific provided context (e.g. retrieved
|
|
24
|
+
* documents), this grounds it against the conversation itself.
|
|
31
25
|
*
|
|
32
26
|
* @param args - The arguments for creating the hallucination evaluator.
|
|
33
|
-
* @
|
|
27
|
+
* @param args.model - The model to use for classification.
|
|
28
|
+
* @param args.choices - The possible classification choices (defaults to hallucinated/grounded).
|
|
29
|
+
* @param args.promptTemplate - The prompt template to use (defaults to HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
30
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
31
|
+
*
|
|
32
|
+
* @returns An evaluator function that takes a {@link HallucinationEvaluationRecord} and returns a classification result
|
|
33
|
+
* indicating whether the response is grounded or hallucinated relative to the conversation.
|
|
34
|
+
*
|
|
35
|
+
* @example
|
|
36
|
+
* ```ts
|
|
37
|
+
* const evaluator = createHallucinationEvaluator({ model: openai("gpt-4o-mini") });
|
|
38
|
+
* const result = await evaluator.evaluate({
|
|
39
|
+
* input:
|
|
40
|
+
* "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
|
|
41
|
+
* output: "Electronics can be returned within 90 days.",
|
|
42
|
+
* });
|
|
43
|
+
* console.log(result.label); // "hallucinated"
|
|
44
|
+
* ```
|
|
34
45
|
*/
|
|
35
46
|
function createHallucinationEvaluator(args) {
|
|
36
|
-
// eslint-disable-next-line no-console
|
|
37
|
-
console.warn("createHallucinationEvaluator is deprecated and will be removed in a future version. " +
|
|
38
|
-
"Please use createFaithfulnessEvaluator instead. The new evaluator uses " +
|
|
39
|
-
"'faithful'/'unfaithful' labels and maximizes score (1.0=faithful) instead of " +
|
|
40
|
-
"minimizing it (0.0=factual).");
|
|
41
47
|
const { choices = default_templates_1.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = default_templates_1.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = default_templates_1.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = default_templates_1.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.name } = args, rest = __rest(args, ["choices", "promptTemplate", "optimizationDirection", "name"]);
|
|
42
48
|
return (0, createClassificationEvaluator_1.createClassificationEvaluator)(Object.assign(Object.assign({}, rest), { promptTemplate,
|
|
43
49
|
choices,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,0EAAmG;AAGnG,mFAAgF;AA6BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,sCAGE,IAA4C;IAE5C,MAAM,EACJ,OAAO,GAAG,iEAA6C,CAAC,OAAO,EAC/D,cAAc,GAAG,iEAA6C,CAAC,QAAQ,EACvE,qBAAqB,GAAG,iEAA6C,CAAC,qBAAqB,EAC3F,IAAI,GAAG,iEAA6C,CAAC,IAAI,KAEvD,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,kCAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,IACJ,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { CreateClassificationEvaluatorArgs } from "../types/evals";
|
|
2
|
+
import type { ClassificationEvaluator } from "./ClassificationEvaluator";
|
|
3
|
+
export interface RetrievalRelevanceEvaluatorArgs<RecordType extends Record<string, unknown> = RetrievalRelevanceEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
|
|
4
|
+
optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
|
|
5
|
+
name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
|
|
6
|
+
choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
|
|
7
|
+
promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
|
|
8
|
+
}
|
|
9
|
+
/** A request and the external information retrieved to serve it. */
|
|
10
|
+
export interface RetrievalRelevanceEvaluationRecord {
|
|
11
|
+
/**
|
|
12
|
+
* The request the retrieval was serving. Prefer the user's request (e.g. the
|
|
13
|
+
* trace root's input) over a reformulated tool argument or generated query.
|
|
14
|
+
*/
|
|
15
|
+
input: string;
|
|
16
|
+
/**
|
|
17
|
+
* The external information retrieved during the step, with all returned items
|
|
18
|
+
* joined together. Source-agnostic: vector search, tool/MCP call, web search,
|
|
19
|
+
* or content embedded in an LLM turn.
|
|
20
|
+
*/
|
|
21
|
+
context: string;
|
|
22
|
+
[key: string]: unknown;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Creates a retrieval relevance evaluator function.
|
|
26
|
+
*
|
|
27
|
+
* This function returns an evaluator that determines whether the external
|
|
28
|
+
* information retrieved during a step is relevant to the request it was
|
|
29
|
+
* serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
|
|
30
|
+
* single document against a question, this evaluator is source-agnostic and
|
|
31
|
+
* scores the retrieved information as a whole (holistically, per retrieval
|
|
32
|
+
* step), whether it came from a vector search, a tool or MCP call, a web
|
|
33
|
+
* search, or content embedded in an LLM turn.
|
|
34
|
+
*
|
|
35
|
+
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
36
|
+
* @param args.model - The model to use for classification.
|
|
37
|
+
* @param args.choices - The possible classification choices (defaults to relevant/irrelevant).
|
|
38
|
+
* @param args.promptTemplate - The prompt template to use (defaults to RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
39
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
40
|
+
*
|
|
41
|
+
* @returns An evaluator function that takes a {@link RetrievalRelevanceEvaluationRecord} and returns a classification result
|
|
42
|
+
* indicating whether the retrieved information is relevant or irrelevant to the request.
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* ```ts
|
|
46
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
47
|
+
* const result = await evaluator.evaluate({
|
|
48
|
+
* input: "What is the capital of France?",
|
|
49
|
+
* context: "Paris is the capital and largest city of France.",
|
|
50
|
+
* });
|
|
51
|
+
* console.log(result.label); // "relevant"
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
export declare function createRetrievalRelevanceEvaluator<RecordType extends Record<string, unknown> = RetrievalRelevanceEvaluationRecord>(args: RetrievalRelevanceEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
|
|
55
|
+
//# sourceMappingURL=createRetrievalRelevanceEvaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __rest = (this && this.__rest) || function (s, e) {
|
|
3
|
+
var t = {};
|
|
4
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
5
|
+
t[p] = s[p];
|
|
6
|
+
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
7
|
+
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
8
|
+
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
9
|
+
t[p[i]] = s[p[i]];
|
|
10
|
+
}
|
|
11
|
+
return t;
|
|
12
|
+
};
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.createRetrievalRelevanceEvaluator = createRetrievalRelevanceEvaluator;
|
|
15
|
+
const default_templates_1 = require("../__generated__/default_templates");
|
|
16
|
+
const createClassificationEvaluator_1 = require("./createClassificationEvaluator");
|
|
17
|
+
/**
|
|
18
|
+
* Creates a retrieval relevance evaluator function.
|
|
19
|
+
*
|
|
20
|
+
* This function returns an evaluator that determines whether the external
|
|
21
|
+
* information retrieved during a step is relevant to the request it was
|
|
22
|
+
* serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
|
|
23
|
+
* single document against a question, this evaluator is source-agnostic and
|
|
24
|
+
* scores the retrieved information as a whole (holistically, per retrieval
|
|
25
|
+
* step), whether it came from a vector search, a tool or MCP call, a web
|
|
26
|
+
* search, or content embedded in an LLM turn.
|
|
27
|
+
*
|
|
28
|
+
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
29
|
+
* @param args.model - The model to use for classification.
|
|
30
|
+
* @param args.choices - The possible classification choices (defaults to relevant/irrelevant).
|
|
31
|
+
* @param args.promptTemplate - The prompt template to use (defaults to RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
32
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
33
|
+
*
|
|
34
|
+
* @returns An evaluator function that takes a {@link RetrievalRelevanceEvaluationRecord} and returns a classification result
|
|
35
|
+
* indicating whether the retrieved information is relevant or irrelevant to the request.
|
|
36
|
+
*
|
|
37
|
+
* @example
|
|
38
|
+
* ```ts
|
|
39
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
40
|
+
* const result = await evaluator.evaluate({
|
|
41
|
+
* input: "What is the capital of France?",
|
|
42
|
+
* context: "Paris is the capital and largest city of France.",
|
|
43
|
+
* });
|
|
44
|
+
* console.log(result.label); // "relevant"
|
|
45
|
+
* ```
|
|
46
|
+
*/
|
|
47
|
+
function createRetrievalRelevanceEvaluator(args) {
|
|
48
|
+
const { choices = default_templates_1.RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = default_templates_1.RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = default_templates_1.RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = default_templates_1.RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.name } = args, rest = __rest(args, ["choices", "promptTemplate", "optimizationDirection", "name"]);
|
|
49
|
+
return (0, createClassificationEvaluator_1.createClassificationEvaluator)(Object.assign(Object.assign({}, rest), { promptTemplate,
|
|
50
|
+
choices,
|
|
51
|
+
optimizationDirection,
|
|
52
|
+
name }));
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=createRetrievalRelevanceEvaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,0EAAyG;AAGzG,mFAAgF;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,2CAIE,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,uEAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,uEAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,uEAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,uEAAmD,CAAC,IAAI,KAE7D,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,kCAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,IACJ,CAAC;AACL,CAAC"}
|
package/dist/src/llm/index.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export * from "./createDocumentRelevanceEvaluator";
|
|
|
7
7
|
export * from "./createFaithfulnessEvaluator";
|
|
8
8
|
export * from "./createHallucinationEvaluator";
|
|
9
9
|
export * from "./createRefusalEvaluator";
|
|
10
|
+
export * from "./createRetrievalRelevanceEvaluator";
|
|
10
11
|
export * from "./createToolInvocationEvaluator";
|
|
11
12
|
export * from "./createToolResponseHandlingEvaluator";
|
|
12
13
|
export * from "./createToolSelectionEvaluator";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
package/dist/src/llm/index.js
CHANGED
|
@@ -21,8 +21,9 @@ __exportStar(require("./createConcisenessEvaluator"), exports);
|
|
|
21
21
|
__exportStar(require("./createCorrectnessEvaluator"), exports);
|
|
22
22
|
__exportStar(require("./createDocumentRelevanceEvaluator"), exports);
|
|
23
23
|
__exportStar(require("./createFaithfulnessEvaluator"), exports);
|
|
24
|
-
__exportStar(require("./createHallucinationEvaluator"), exports);
|
|
24
|
+
__exportStar(require("./createHallucinationEvaluator"), exports);
|
|
25
25
|
__exportStar(require("./createRefusalEvaluator"), exports);
|
|
26
|
+
__exportStar(require("./createRetrievalRelevanceEvaluator"), exports);
|
|
26
27
|
__exportStar(require("./createToolInvocationEvaluator"), exports);
|
|
27
28
|
__exportStar(require("./createToolResponseHandlingEvaluator"), exports);
|
|
28
29
|
__exportStar(require("./createToolSelectionEvaluator"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4DAA0C;AAC1C,kEAAgD;AAChD,uDAAqC;AACrC,+DAA6C;AAC7C,+DAA6C;AAC7C,qEAAmD;AACnD,gEAA8C;AAC9C,iEAA+C
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4DAA0C;AAC1C,kEAAgD;AAChD,uDAAqC;AACrC,+DAA6C;AAC7C,+DAA6C;AAC7C,qEAAmD;AACnD,gEAA8C;AAC9C,iEAA+C;AAC/C,2DAAyC;AACzC,sEAAoD;AACpD,kEAAgD;AAChD,wEAAsD;AACtD,iEAA+C;AAC/C,4DAA0C;AAC1C,gEAA8C;AAC9C,2DAAyC;AACzC,iDAA+B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getTemplateVariables.js","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":";;;;;;AAAA,wDAAgC;AAOhC;;;;GAIG;AACH,8BAAqC,IAA6B;IAChE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC1B,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,8BAA8B,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,QAAQ,CAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"getTemplateVariables.js","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":";;;;;;AAAA,wDAAgC;AAOhC;;;;GAIG;AACH,8BAAqC,IAA6B;IAChE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC1B,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,8BAA8B,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,QAAQ,CAAC,MAAM,CAAW,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;QAChD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,CAAC,GAAG,GAAG,EAAE,GAAG,8BAA8B,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAE,CAAC,CAAC;AACT,CAAC;AACD;;;;GAIG;AACH,SAAS,8BAA8B,CAAC,QAAgB;IACtD,MAAM,aAAa,GAAG,kBAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,aAAa,CAAC,MAAM,CAAW,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE;QAC1D,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,YAAY,CAAC;QACvC,IAAI,QAAQ,KAAK,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrD,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAE,CAAC,CAAC;AACT,CAAC"}
|