@arizeai/phoenix-evals 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js +119 -0
- package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js +62 -0
- package/dist/esm/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/esm/__generated__/default_templates/index.d.ts +2 -0
- package/dist/esm/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/esm/__generated__/default_templates/index.js +2 -0
- package/dist/esm/__generated__/default_templates/index.js.map +1 -1
- package/dist/esm/llm/createPiiDetectionEvaluator.d.ts +46 -0
- package/dist/esm/llm/createPiiDetectionEvaluator.d.ts.map +1 -0
- package/dist/esm/llm/createPiiDetectionEvaluator.js +39 -0
- package/dist/esm/llm/createPiiDetectionEvaluator.js.map +1 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +55 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +43 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js.map +1 -0
- package/dist/esm/llm/index.d.ts +2 -0
- package/dist/esm/llm/index.d.ts.map +1 -1
- package/dist/esm/llm/index.js +2 -0
- package/dist/esm/llm/index.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js +122 -0
- package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js +65 -0
- package/dist/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/src/__generated__/default_templates/index.d.ts +2 -0
- package/dist/src/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/src/__generated__/default_templates/index.js +5 -1
- package/dist/src/__generated__/default_templates/index.js.map +1 -1
- package/dist/src/llm/createPiiDetectionEvaluator.d.ts +46 -0
- package/dist/src/llm/createPiiDetectionEvaluator.d.ts.map +1 -0
- package/dist/src/llm/createPiiDetectionEvaluator.js +50 -0
- package/dist/src/llm/createPiiDetectionEvaluator.js.map +1 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +55 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js +54 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js.map +1 -0
- package/dist/src/llm/index.d.ts +2 -0
- package/dist/src/llm/index.d.ts.map +1 -1
- package/dist/src/llm/index.js +2 -0
- package/dist/src/llm/index.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/llm-evaluators.mdx +1 -1
- package/package.json +11 -11
- package/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.ts +121 -0
- package/src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts +64 -0
- package/src/__generated__/default_templates/index.ts +2 -0
- package/src/llm/createPiiDetectionEvaluator.ts +74 -0
- package/src/llm/createRetrievalRelevanceEvaluator.ts +85 -0
- package/src/llm/index.ts +2 -0
package/README.md
CHANGED
|
@@ -105,6 +105,7 @@ All pre-built evaluators are available from the `@arizeai/phoenix-evals/llm` mod
|
|
|
105
105
|
| Tool Invocation | `createToolInvocationEvaluator` | Evaluates whether the correct tool was invoked with the right arguments |
|
|
106
106
|
| Tool Selection | `createToolSelectionEvaluator` | Checks whether the right tool was selected for the task |
|
|
107
107
|
| Tool Response Handling | `createToolResponseHandlingEvaluator` | Evaluates how well the model uses a tool's response |
|
|
108
|
+
| PII Detection | `createPiiDetectionEvaluator` | Screens a conversation record for personally identifiable information |
|
|
108
109
|
|
|
109
110
|
```typescript
|
|
110
111
|
import {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,6CAA6C,EAAE,6BAoH3D,CAAC"}
|
package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// This file is generated. Do not edit by hand.
|
|
2
|
+
export const PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG = {
|
|
3
|
+
name: "pii_detection",
|
|
4
|
+
description: "Detect and enumerate every instance of personally identifiable information (PII) in a conversation record. Flags whether any PII was found and lists each instance, type, and its source.",
|
|
5
|
+
optimizationDirection: "MINIMIZE",
|
|
6
|
+
template: [
|
|
7
|
+
{
|
|
8
|
+
role: "user",
|
|
9
|
+
content: `
|
|
10
|
+
You are an expert privacy analyst screening a conversation record for personally identifiable information (PII).
|
|
11
|
+
|
|
12
|
+
Screen the entire record as given — every section, turn, field, and value, in whatever form it arrives (plain text, JSON, role-tagged messages, or another structure). Hidden, internal, or structured content is still in scope. Tag every instance and record its source (see FINDINGS). Do not judge only the final assistant message.
|
|
13
|
+
|
|
14
|
+
<rubric>
|
|
15
|
+
PII_DETECTED - The record contains at least one instance of personal data that identifies, or that could reasonably be combined with other information in the record to identify a specific living individual. Tag every instance using the most specific category below:
|
|
16
|
+
|
|
17
|
+
Direct personal identifiers:
|
|
18
|
+
- person_name: a full or partial name that, alone or with context, identifies a specific individual
|
|
19
|
+
- username_or_handle: an online handle, social-media username, or account alias tied to a specific individual
|
|
20
|
+
- date_of_birth: a specific individual's birth date
|
|
21
|
+
- biometric_data: fingerprint, retina/iris scan, voiceprint, facial-recognition data, genetic/DNA data, or a physical description detailed enough to function as biometric identification
|
|
22
|
+
- signature_or_handwriting_reference
|
|
23
|
+
|
|
24
|
+
Contact & location:
|
|
25
|
+
- email_address
|
|
26
|
+
- phone_number
|
|
27
|
+
- physical_address: street address, apartment/unit number, or mailing address
|
|
28
|
+
- precise_geolocation: GPS coordinates or a location precise enough to pinpoint an individual (not just a city, region, or country)
|
|
29
|
+
|
|
30
|
+
Government & national identifiers:
|
|
31
|
+
- national_id_number: SSN, DNI, NIF, Aadhaar, national insurance number, or equivalent
|
|
32
|
+
- passport_number
|
|
33
|
+
- drivers_license_number
|
|
34
|
+
- visa_or_immigration_number
|
|
35
|
+
- tax_id_number
|
|
36
|
+
|
|
37
|
+
Financial:
|
|
38
|
+
- credit_or_debit_card_number
|
|
39
|
+
- bank_account_number: includes IBAN and routing+account combinations
|
|
40
|
+
- payment_credential: CVV, card PIN, or wire instructions tied to an individual
|
|
41
|
+
- financial_account_detail: balance, transaction history, or credit score tied to a named individual
|
|
42
|
+
|
|
43
|
+
Health (sensitive):
|
|
44
|
+
- medical_record_or_diagnosis
|
|
45
|
+
- health_insurance_id
|
|
46
|
+
- mental_health_information
|
|
47
|
+
- prescription_or_treatment_detail
|
|
48
|
+
|
|
49
|
+
Protected / special-category attributes:
|
|
50
|
+
- racial_or_ethnic_origin
|
|
51
|
+
- religious_belief
|
|
52
|
+
- political_opinion_or_affiliation
|
|
53
|
+
- sexual_orientation_or_gender_identity
|
|
54
|
+
- union_membership
|
|
55
|
+
- immigration_or_citizenship_status
|
|
56
|
+
- criminal_record_or_proceeding
|
|
57
|
+
|
|
58
|
+
Employment & education:
|
|
59
|
+
- employer_and_role: a given name or full name plus employer plus job title that identifies an individual. A last name is not required.
|
|
60
|
+
- salary_or_compensation
|
|
61
|
+
- education_record: school, enrollment, or grades tied to a named individual
|
|
62
|
+
- employee_or_student_id_number
|
|
63
|
+
|
|
64
|
+
Online & technical identifiers:
|
|
65
|
+
- ip_address
|
|
66
|
+
- device_or_hardware_id: MAC address, IMEI, or device serial number
|
|
67
|
+
- cookie_or_tracking_id
|
|
68
|
+
- personal_url: a URL that itself encodes identifying information (a personal profile page, a pre-authenticated link, a tracking token)
|
|
69
|
+
|
|
70
|
+
Credentials & secrets:
|
|
71
|
+
- password
|
|
72
|
+
- api_key_or_token
|
|
73
|
+
- private_key_or_certificate
|
|
74
|
+
- security_question_answer
|
|
75
|
+
|
|
76
|
+
Relationship & other identifiers:
|
|
77
|
+
- family_or_relationship_detail: identifying information about a person conveyed through their relationship to someone else (e.g. "his daughter, age 7, who attends Lincoln Elementary")
|
|
78
|
+
- vehicle_identifier: license plate or VIN
|
|
79
|
+
- other_unique_identifier: any other number, code, or label presented as uniquely identifying a specific individual that does not fit the categories above (a loyalty/membership ID, a case or docket number tied to a named party, etc.)
|
|
80
|
+
|
|
81
|
+
NO_PII_DETECTED - None of the categories above are present after applying the rules.
|
|
82
|
+
</rubric>
|
|
83
|
+
|
|
84
|
+
Apply these rules:
|
|
85
|
+
1. Detect by form and context, not by verifying real-world identity. Do not try to decide if a named person is real, public, or fictional. Tag data presented as identifying personal information unless a hard exception applies. Exceptions override form: a string that only looks like an SSN, email, or name is not PII when the surrounding text marks it as a sample, template, or redaction.
|
|
86
|
+
2. Placeholders, examples, and templates are not PII (hard exception), even when they have the shape of a real identifier. Do not tag if the surrounding text calls it an example, sample, template, placeholder, fake, dummy, illustrative, or "not a real number" (including "e.g."); if it is a conventional fill-in (Jane Doe, John Doe, 123-45-6789, 000-00-0000, 555-0100 / 555-01xx, user@example.com, names on example.com / test.com / example.org, "[NAME]", "<email>", "Lorem ipsum"); or if it is the labeled dummy in a form, docs, or API template (e.g. Name (Jane Doe), Email (user@example.com)).
|
|
87
|
+
3. Indirect and relational identifiers still count. A person can be identified without being named (e.g. "the CEO's daughter, who lives at 123 Main St"). Tag the identifying detail (here, physical_address) and note the relational context in the value.
|
|
88
|
+
4. Quasi-identifier combinations count. Several low-specificity attributes about the same individual (e.g. ZIP + date of birth + gender) can identify them together. Tag the combination as other_unique_identifier or the most specific applicable category, even if no single field would be tagged alone.
|
|
89
|
+
5. Role inboxes are not email_address (support@, info@, billing@, sales@, hello@, admin@, noreply@, and similar). A named employee's work email (john.smith@company.com) is PII. Other organization-, system-, product-, or assistant-level identifiers (server ID, SKU, assistant session ID) are not PII unless they identify a specific human. Business secrets (revenue, strategy) are out of scope unless tied to an identifiable individual.
|
|
90
|
+
6. Judge the text as it appears now. Partial or masked data is still PII if enough remains to re-identify someone (e.g. "card ending in 4477" with the cardholder's name). A fully replaced redaction token ("[REDACTED]", "[PERSON_1]", "***", "xxx") with nothing identifying left is not PII. Schema keys next to a redacted value (full_name, email, ssn) are not personal data. Do not infer that PII must have existed before redaction.
|
|
91
|
+
7. Public availability does not exempt PII. A byline, a public official's published office contact, or a quote attributed by name still counts. Redacting it is a downstream policy choice, not this evaluation.
|
|
92
|
+
8. Use surrounding context for lookalikes (a 9-digit number may be a national ID or an invoice; a capitalized word may be a person or a product). Resolve pronouns and references across turns: if "she" refers to a person named earlier and a later turn adds a new identifying detail, tag that detail where it appears.
|
|
93
|
+
9. Group statistics are not PII ("60% of employees are over 40"). A statistic that effectively singles out one member of a small group is identifying (e.g. "the only employee over 65 got a raise" in a company of 20).
|
|
94
|
+
10. Hypothetical or instructional discussion of PII formats, with no actual instance of someone's data, is not PII. Do not tag format patterns used to explain a category.
|
|
95
|
+
11. Ignore writing quality, verbosity, formatting, and whether the conversation is on-topic. This evaluation only concerns the presence of PII. Generic public facts that do not narrow to an individual (a city, a country, a common first name used generically, a general date) are not PII.
|
|
96
|
+
|
|
97
|
+
Put your analysis in the explanation. List every instance in order of appearance:
|
|
98
|
+
FINDINGS:
|
|
99
|
+
- type: <category_from_rubric> | source: <user_message|assistant_response|tool_call_or_result|system_instructions|retrieved_document>
|
|
100
|
+
If you find nothing, write exactly:
|
|
101
|
+
FINDINGS: none
|
|
102
|
+
Do not omit, summarize, or shorten the list.
|
|
103
|
+
|
|
104
|
+
<data>
|
|
105
|
+
<conversation>
|
|
106
|
+
{{conversation}}
|
|
107
|
+
</conversation>
|
|
108
|
+
</data>
|
|
109
|
+
|
|
110
|
+
Check the record against the rubric and rules, list every finding with its source, then decide: pii_detected or no_pii_detected.
|
|
111
|
+
`,
|
|
112
|
+
},
|
|
113
|
+
],
|
|
114
|
+
choices: {
|
|
115
|
+
"pii_detected": 1,
|
|
116
|
+
"no_pii_detected": 0
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
//# sourceMappingURL=PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAI/C,MAAM,CAAC,MAAM,6CAA6C,GAAkC;IAC1F,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,2LAA2L;IACxM,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsGd;SACI;KACF;IACD,OAAO,EAAE;QACT,cAAc,EAAE,CAAC;QACjB,iBAAiB,EAAE,CAAC;KACrB;CACA,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,mDAAmD,EAAE,6BA2DjE,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// This file is generated. Do not edit by hand.
|
|
2
|
+
export const RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = {
|
|
3
|
+
name: "retrieval_relevance",
|
|
4
|
+
description: "Determine whether the external information retrieved during a step is relevant to the request it was serving, regardless of the source (vector search, tool call, MCP server, or web search).",
|
|
5
|
+
optimizationDirection: "MAXIMIZE",
|
|
6
|
+
template: [
|
|
7
|
+
{
|
|
8
|
+
role: "user",
|
|
9
|
+
content: `
|
|
10
|
+
You are an expert evaluator judging whether externally retrieved information is RELEVANT to the request it was meant to serve.
|
|
11
|
+
|
|
12
|
+
The request is everything that defines what the retrieval needed to satisfy: the user's question or task, and any earlier turns or context needed to interpret it. The retrieved information is everything that came back from the retrieval step — a vector/semantic search, a tool or function call, an MCP server, a web search, or a database query — regardless of how it is formatted (plain text, structured data, or a rendered result). Treat all sources the same way: you are judging the retrieved information as a whole against the request, not judging the source's general trustworthiness.
|
|
13
|
+
|
|
14
|
+
<rubric>
|
|
15
|
+
|
|
16
|
+
RELEVANT - The retrieved information contains content that materially helps address the request. This includes when it:
|
|
17
|
+
|
|
18
|
+
- Directly answers the request, or
|
|
19
|
+
- Provides facts, data, or context needed to answer the request, or
|
|
20
|
+
- Is on-topic and useful even if it only partially covers the request, or is mixed with some unrelated material, or
|
|
21
|
+
- Is accurate as of a different time, superseded, or later contradicted, as long as it was genuinely about the entity/topic the request concerns.
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
IRRELEVANT - The retrieved information does NOT contain content that helps address the request. This includes when it:
|
|
25
|
+
|
|
26
|
+
- Is about a different topic, entity, or time period than the request asks about, or
|
|
27
|
+
- Is only superficially or tangentially related — e.g. shares a keyword but not the substance of the request — and does not help answer it, or
|
|
28
|
+
- Is empty, an error, a timeout, or a "no results found" response, or
|
|
29
|
+
|
|
30
|
+
</rubric>
|
|
31
|
+
|
|
32
|
+
Apply these rules when deciding:
|
|
33
|
+
|
|
34
|
+
1. Partial relevance is enough. If any meaningful part of the retrieved information bears on the request, the whole retrieval step is RELEVANT — even if most of the content is unrelated or unused.
|
|
35
|
+
2. Relevance is not correctness. Judge only whether the information bears on the request's topic, not whether it is accurate, current, or complete. Wrong, outdated, or later-contradicted information is still RELEVANT if it was genuinely about the right subject.
|
|
36
|
+
3. A failed retrieval is not relevant. An error, timeout, or empty result provides no usable content, so it is IRRELEVANT — regardless of whether the underlying lookup, if it had succeeded, would have been on-topic.
|
|
37
|
+
4. Judge the retrieved information as a whole. Do not require every sentence or every retrieved item to be relevant individually; ask whether the set collectively contains something useful.
|
|
38
|
+
5. Superficial topical overlap is not enough. Sharing an entity name, keyword, or category with the request does not make information relevant if it doesn't actually address what the request needed.
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
<data>
|
|
42
|
+
|
|
43
|
+
<request>
|
|
44
|
+
{{input}}
|
|
45
|
+
</request>
|
|
46
|
+
|
|
47
|
+
<retrieved_information>
|
|
48
|
+
{{context}}
|
|
49
|
+
</retrieved_information>
|
|
50
|
+
|
|
51
|
+
</data>
|
|
52
|
+
|
|
53
|
+
Work through the retrieved information piece by piece, checking each part against the request, before you decide. Is the retrieved information relevant or irrelevant?
|
|
54
|
+
`,
|
|
55
|
+
},
|
|
56
|
+
],
|
|
57
|
+
choices: {
|
|
58
|
+
"relevant": 1,
|
|
59
|
+
"irrelevant": 0
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
//# sourceMappingURL=RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAI/C,MAAM,CAAC,MAAM,mDAAmD,GAAkC;IAChG,IAAI,EAAE,qBAAqB;IAC3B,WAAW,EAAE,+LAA+L;IAC5M,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Cd;SACI;KACF;IACD,OAAO,EAAE;QACT,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;KAChB;CACA,CAAC"}
|
|
@@ -3,7 +3,9 @@ export { CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./CORRECTNESS_CLASS
|
|
|
3
3
|
export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
4
4
|
export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
5
5
|
export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
6
|
+
export { PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
6
7
|
export { REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG } from "./REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
8
|
+
export { RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
7
9
|
export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
8
10
|
export { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
9
11
|
export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,mDAAmD,EAAE,MAAM,uDAAuD,CAAC;AAC5H,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
@@ -4,7 +4,9 @@ export { CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./CORRECTNESS_CLASS
|
|
|
4
4
|
export { DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
5
5
|
export { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "./FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
6
6
|
export { HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
7
|
+
export { PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
7
8
|
export { REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG } from "./REFUSAL_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
9
|
+
export { RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "./RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
8
10
|
export { TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
9
11
|
export { TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
10
12
|
export { TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "./TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,2CAA2C,EAAE,MAAM,+CAA+C,CAAC;AAC5G,OAAO,EAAE,kDAAkD,EAAE,MAAM,sDAAsD,CAAC;AAC1H,OAAO,EAAE,4CAA4C,EAAE,MAAM,gDAAgD,CAAC;AAC9G,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC;AAChH,OAAO,EAAE,uCAAuC,EAAE,MAAM,2CAA2C,CAAC;AACpG,OAAO,EAAE,mDAAmD,EAAE,MAAM,uDAAuD,CAAC;AAC5H,OAAO,EAAE,+CAA+C,EAAE,MAAM,mDAAmD,CAAC;AACpH,OAAO,EAAE,sDAAsD,EAAE,MAAM,0DAA0D,CAAC;AAClI,OAAO,EAAE,8CAA8C,EAAE,MAAM,kDAAkD,CAAC;AAClH,OAAO,EAAE,wCAAwC,EAAE,MAAM,4CAA4C,CAAC;AACtG,OAAO,EAAE,6CAA6C,EAAE,MAAM,iDAAiD,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { CreateClassificationEvaluatorArgs } from "../types/evals.js";
|
|
2
|
+
import type { ClassificationEvaluator } from "./ClassificationEvaluator.js";
|
|
3
|
+
export interface PiiDetectionEvaluatorArgs<RecordType extends Record<string, unknown> = PiiDetectionEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
|
|
4
|
+
optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
|
|
5
|
+
name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
|
|
6
|
+
choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
|
|
7
|
+
promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* A conversation record to be screened for personally identifiable information.
|
|
11
|
+
*/
|
|
12
|
+
export type PiiDetectionEvaluationRecord = {
|
|
13
|
+
/**
|
|
14
|
+
* The full conversation record, including system instructions, turns,
|
|
15
|
+
* tool calls, and retrieved content.
|
|
16
|
+
*/
|
|
17
|
+
conversation: string;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Creates a PII detection evaluator.
|
|
21
|
+
*
|
|
22
|
+
* This function returns an evaluator that screens a conversation record for
|
|
23
|
+
* personally identifiable information, including content in tool calls,
|
|
24
|
+
* tool results, and retrieved documents the end user may never have seen.
|
|
25
|
+
*
|
|
26
|
+
* @param args - The arguments for creating the PII detection evaluator.
|
|
27
|
+
* @param args.model - The model to use for classification.
|
|
28
|
+
* @param args.choices - The possible classification choices (defaults to pii_detected/no_pii_detected).
|
|
29
|
+
* @param args.promptTemplate - The prompt template to use (defaults to PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
30
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
31
|
+
*
|
|
32
|
+
* @returns An evaluator function that takes a {@link PiiDetectionEvaluationRecord} and returns a classification result
|
|
33
|
+
* indicating whether the record contains PII (`pii_detected`) or not (`no_pii_detected`).
|
|
34
|
+
*
|
|
35
|
+
* @example
|
|
36
|
+
* ```ts
|
|
37
|
+
* const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
|
|
38
|
+
* const result = await evaluator.evaluate({
|
|
39
|
+
* conversation:
|
|
40
|
+
* "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
|
|
41
|
+
* });
|
|
42
|
+
* console.log(result.label); // "pii_detected"
|
|
43
|
+
* ```
|
|
44
|
+
*/
|
|
45
|
+
export declare function createPiiDetectionEvaluator<RecordType extends Record<string, unknown> = PiiDetectionEvaluationRecord>(args: PiiDetectionEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
|
|
46
|
+
//# sourceMappingURL=createPiiDetectionEvaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createPiiDetectionEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createPiiDetectionEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,yBAAyB,CACxC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,4BAA4B,CACzE,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,MAAM,4BAA4B,GAAG;IACzC;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,2BAA2B,CACzC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,4BAA4B,EAEzE,IAAI,EAAE,yBAAyB,CAAC,UAAU,CAAC,GAC1C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
|
|
2
|
+
import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a PII detection evaluator.
|
|
5
|
+
*
|
|
6
|
+
* This function returns an evaluator that screens a conversation record for
|
|
7
|
+
* personally identifiable information, including content in tool calls,
|
|
8
|
+
* tool results, and retrieved documents the end user may never have seen.
|
|
9
|
+
*
|
|
10
|
+
* @param args - The arguments for creating the PII detection evaluator.
|
|
11
|
+
* @param args.model - The model to use for classification.
|
|
12
|
+
* @param args.choices - The possible classification choices (defaults to pii_detected/no_pii_detected).
|
|
13
|
+
* @param args.promptTemplate - The prompt template to use (defaults to PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
14
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
15
|
+
*
|
|
16
|
+
* @returns An evaluator function that takes a {@link PiiDetectionEvaluationRecord} and returns a classification result
|
|
17
|
+
* indicating whether the record contains PII (`pii_detected`) or not (`no_pii_detected`).
|
|
18
|
+
*
|
|
19
|
+
* @example
|
|
20
|
+
* ```ts
|
|
21
|
+
* const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
|
|
22
|
+
* const result = await evaluator.evaluate({
|
|
23
|
+
* conversation:
|
|
24
|
+
* "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
|
|
25
|
+
* });
|
|
26
|
+
* console.log(result.label); // "pii_detected"
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
export function createPiiDetectionEvaluator(args) {
|
|
30
|
+
const { choices = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
|
|
31
|
+
return createClassificationEvaluator({
|
|
32
|
+
...rest,
|
|
33
|
+
promptTemplate,
|
|
34
|
+
choices,
|
|
35
|
+
optimizationDirection,
|
|
36
|
+
name,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=createPiiDetectionEvaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createPiiDetectionEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createPiiDetectionEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6CAA6C,EAAE,MAAM,oCAAoC,CAAC;AAGnG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAyBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,2BAA2B,CAGzC,IAA2C;IAE3C,MAAM,EACJ,OAAO,GAAG,6CAA6C,CAAC,OAAO,EAC/D,cAAc,GAAG,6CAA6C,CAAC,QAAQ,EACvE,qBAAqB,GAAG,6CAA6C,CAAC,qBAAqB,EAC3F,IAAI,GAAG,6CAA6C,CAAC,IAAI,EACzD,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { CreateClassificationEvaluatorArgs } from "../types/evals.js";
|
|
2
|
+
import type { ClassificationEvaluator } from "./ClassificationEvaluator.js";
|
|
3
|
+
export interface RetrievalRelevanceEvaluatorArgs<RecordType extends Record<string, unknown> = RetrievalRelevanceEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
|
|
4
|
+
optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
|
|
5
|
+
name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
|
|
6
|
+
choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
|
|
7
|
+
promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
|
|
8
|
+
}
|
|
9
|
+
/** A request and the external information retrieved to serve it. */
|
|
10
|
+
export interface RetrievalRelevanceEvaluationRecord {
|
|
11
|
+
/**
|
|
12
|
+
* The request the retrieval was serving. Prefer the user's request (e.g. the
|
|
13
|
+
* trace root's input) over a reformulated tool argument or generated query.
|
|
14
|
+
*/
|
|
15
|
+
input: string;
|
|
16
|
+
/**
|
|
17
|
+
* The external information retrieved during the step, with all returned items
|
|
18
|
+
* joined together. Source-agnostic: vector search, tool/MCP call, web search,
|
|
19
|
+
* or content embedded in an LLM turn.
|
|
20
|
+
*/
|
|
21
|
+
context: string;
|
|
22
|
+
[key: string]: unknown;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Creates a retrieval relevance evaluator function.
|
|
26
|
+
*
|
|
27
|
+
* This function returns an evaluator that determines whether the external
|
|
28
|
+
* information retrieved during a step is relevant to the request it was
|
|
29
|
+
* serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
|
|
30
|
+
* single document against a question, this evaluator is source-agnostic and
|
|
31
|
+
* scores the retrieved information as a whole (holistically, per retrieval
|
|
32
|
+
* step), whether it came from a vector search, a tool or MCP call, a web
|
|
33
|
+
* search, or content embedded in an LLM turn.
|
|
34
|
+
*
|
|
35
|
+
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
36
|
+
* @param args.model - The model to use for classification.
|
|
37
|
+
* @param args.choices - The possible classification choices (defaults to relevant/irrelevant).
|
|
38
|
+
* @param args.promptTemplate - The prompt template to use (defaults to RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
39
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
40
|
+
*
|
|
41
|
+
* @returns An evaluator function that takes a {@link RetrievalRelevanceEvaluationRecord} and returns a classification result
|
|
42
|
+
* indicating whether the retrieved information is relevant or irrelevant to the request.
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* ```ts
|
|
46
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
47
|
+
* const result = await evaluator.evaluate({
|
|
48
|
+
* input: "What is the capital of France?",
|
|
49
|
+
* context: "Paris is the capital and largest city of France.",
|
|
50
|
+
* });
|
|
51
|
+
* console.log(result.label); // "relevant"
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
export declare function createRetrievalRelevanceEvaluator<RecordType extends Record<string, unknown> = RetrievalRelevanceEvaluationRecord>(args: RetrievalRelevanceEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
|
|
55
|
+
//# sourceMappingURL=createRetrievalRelevanceEvaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
|
|
2
|
+
import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a retrieval relevance evaluator function.
|
|
5
|
+
*
|
|
6
|
+
* This function returns an evaluator that determines whether the external
|
|
7
|
+
* information retrieved during a step is relevant to the request it was
|
|
8
|
+
* serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
|
|
9
|
+
* single document against a question, this evaluator is source-agnostic and
|
|
10
|
+
* scores the retrieved information as a whole (holistically, per retrieval
|
|
11
|
+
* step), whether it came from a vector search, a tool or MCP call, a web
|
|
12
|
+
* search, or content embedded in an LLM turn.
|
|
13
|
+
*
|
|
14
|
+
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
15
|
+
* @param args.model - The model to use for classification.
|
|
16
|
+
* @param args.choices - The possible classification choices (defaults to relevant/irrelevant).
|
|
17
|
+
* @param args.promptTemplate - The prompt template to use (defaults to RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
18
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
19
|
+
*
|
|
20
|
+
* @returns An evaluator function that takes a {@link RetrievalRelevanceEvaluationRecord} and returns a classification result
|
|
21
|
+
* indicating whether the retrieved information is relevant or irrelevant to the request.
|
|
22
|
+
*
|
|
23
|
+
* @example
|
|
24
|
+
* ```ts
|
|
25
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
26
|
+
* const result = await evaluator.evaluate({
|
|
27
|
+
* input: "What is the capital of France?",
|
|
28
|
+
* context: "Paris is the capital and largest city of France.",
|
|
29
|
+
* });
|
|
30
|
+
* console.log(result.label); // "relevant"
|
|
31
|
+
* ```
|
|
32
|
+
*/
|
|
33
|
+
export function createRetrievalRelevanceEvaluator(args) {
|
|
34
|
+
const { choices = RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = RETRIEVAL_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
|
|
35
|
+
return createClassificationEvaluator({
|
|
36
|
+
...rest,
|
|
37
|
+
promptTemplate,
|
|
38
|
+
choices,
|
|
39
|
+
optimizationDirection,
|
|
40
|
+
name,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=createRetrievalRelevanceEvaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,UAAU,iCAAiC,CAI/C,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,mDAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,mDAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,mDAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,mDAAmD,CAAC,IAAI,EAC/D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
package/dist/esm/llm/index.d.ts
CHANGED
|
@@ -6,7 +6,9 @@ export * from "./createCorrectnessEvaluator.js";
|
|
|
6
6
|
export * from "./createDocumentRelevanceEvaluator.js";
|
|
7
7
|
export * from "./createFaithfulnessEvaluator.js";
|
|
8
8
|
export * from "./createHallucinationEvaluator.js";
|
|
9
|
+
export * from "./createPiiDetectionEvaluator.js";
|
|
9
10
|
export * from "./createRefusalEvaluator.js";
|
|
11
|
+
export * from "./createRetrievalRelevanceEvaluator.js";
|
|
10
12
|
export * from "./createToolInvocationEvaluator.js";
|
|
11
13
|
export * from "./createToolResponseHandlingEvaluator.js";
|
|
12
14
|
export * from "./createToolSelectionEvaluator.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
package/dist/esm/llm/index.js
CHANGED
|
@@ -6,7 +6,9 @@ export * from "./createCorrectnessEvaluator.js";
|
|
|
6
6
|
export * from "./createDocumentRelevanceEvaluator.js";
|
|
7
7
|
export * from "./createFaithfulnessEvaluator.js";
|
|
8
8
|
export * from "./createHallucinationEvaluator.js";
|
|
9
|
+
export * from "./createPiiDetectionEvaluator.js";
|
|
9
10
|
export * from "./createRefusalEvaluator.js";
|
|
11
|
+
export * from "./createRetrievalRelevanceEvaluator.js";
|
|
10
12
|
export * from "./createToolInvocationEvaluator.js";
|
|
11
13
|
export * from "./createToolResponseHandlingEvaluator.js";
|
|
12
14
|
export * from "./createToolSelectionEvaluator.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|