@arizeai/phoenix-evals 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +256 -0
- package/README.md +7 -5
- package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js +93 -0
- package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js +119 -0
- package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/esm/__generated__/default_templates/index.d.ts +2 -0
- package/dist/esm/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/esm/__generated__/default_templates/index.js +2 -0
- package/dist/esm/__generated__/default_templates/index.js.map +1 -1
- package/dist/esm/llm/createCompletenessEvaluator.d.ts +46 -0
- package/dist/esm/llm/createCompletenessEvaluator.d.ts.map +1 -0
- package/dist/esm/llm/createCompletenessEvaluator.js +38 -0
- package/dist/esm/llm/createCompletenessEvaluator.js.map +1 -0
- package/dist/esm/llm/createDocumentRelevanceEvaluator.d.ts +16 -11
- package/dist/esm/llm/createDocumentRelevanceEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/createDocumentRelevanceEvaluator.js +7 -11
- package/dist/esm/llm/createDocumentRelevanceEvaluator.js.map +1 -1
- package/dist/esm/llm/createPiiDetectionEvaluator.d.ts +46 -0
- package/dist/esm/llm/createPiiDetectionEvaluator.d.ts.map +1 -0
- package/dist/esm/llm/createPiiDetectionEvaluator.js +39 -0
- package/dist/esm/llm/createPiiDetectionEvaluator.js.map +1 -0
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +4 -5
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +4 -5
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js.map +1 -1
- package/dist/esm/llm/index.d.ts +2 -0
- package/dist/esm/llm/index.d.ts.map +1 -1
- package/dist/esm/llm/index.js +2 -0
- package/dist/esm/llm/index.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/esm/utils/bindEvaluator.d.ts +4 -5
- package/dist/esm/utils/bindEvaluator.d.ts.map +1 -1
- package/dist/esm/utils/bindEvaluator.js +4 -5
- package/dist/esm/utils/bindEvaluator.js.map +1 -1
- package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js +96 -0
- package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
- package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
- package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js +122 -0
- package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
- package/dist/src/__generated__/default_templates/index.d.ts +2 -0
- package/dist/src/__generated__/default_templates/index.d.ts.map +1 -1
- package/dist/src/__generated__/default_templates/index.js +5 -1
- package/dist/src/__generated__/default_templates/index.js.map +1 -1
- package/dist/src/llm/createCompletenessEvaluator.d.ts +46 -0
- package/dist/src/llm/createCompletenessEvaluator.d.ts.map +1 -0
- package/dist/src/llm/createCompletenessEvaluator.js +49 -0
- package/dist/src/llm/createCompletenessEvaluator.js.map +1 -0
- package/dist/src/llm/createDocumentRelevanceEvaluator.d.ts +16 -11
- package/dist/src/llm/createDocumentRelevanceEvaluator.d.ts.map +1 -1
- package/dist/src/llm/createDocumentRelevanceEvaluator.js +7 -11
- package/dist/src/llm/createDocumentRelevanceEvaluator.js.map +1 -1
- package/dist/src/llm/createPiiDetectionEvaluator.d.ts +46 -0
- package/dist/src/llm/createPiiDetectionEvaluator.d.ts.map +1 -0
- package/dist/src/llm/createPiiDetectionEvaluator.js +50 -0
- package/dist/src/llm/createPiiDetectionEvaluator.js.map +1 -0
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +4 -5
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js +4 -5
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js.map +1 -1
- package/dist/src/llm/index.d.ts +2 -0
- package/dist/src/llm/index.d.ts.map +1 -1
- package/dist/src/llm/index.js +2 -0
- package/dist/src/llm/index.js.map +1 -1
- package/dist/src/utils/bindEvaluator.d.ts +4 -5
- package/dist/src/utils/bindEvaluator.d.ts.map +1 -1
- package/dist/src/utils/bindEvaluator.js +4 -5
- package/dist/src/utils/bindEvaluator.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/llm-evaluators.mdx +1 -1
- package/package.json +12 -12
- package/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.ts +95 -0
- package/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.ts +121 -0
- package/src/__generated__/default_templates/index.ts +2 -0
- package/src/llm/createCompletenessEvaluator.ts +74 -0
- package/src/llm/createDocumentRelevanceEvaluator.ts +16 -11
- package/src/llm/createPiiDetectionEvaluator.ts +74 -0
- package/src/llm/createRetrievalRelevanceEvaluator.ts +4 -5
- package/src/llm/index.ts +2 -0
- package/src/utils/bindEvaluator.ts +4 -5
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
|
|
2
|
+
import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a completeness evaluator.
|
|
5
|
+
*
|
|
6
|
+
* This function returns an evaluator that checks whether every active user
|
|
7
|
+
* request in a conversation was actually completed, not merely acknowledged.
|
|
8
|
+
*
|
|
9
|
+
* @param args - The arguments for creating the completeness evaluator.
|
|
10
|
+
* @param args.model - The model to use for classification.
|
|
11
|
+
* @param args.choices - The possible classification choices (defaults to complete/incomplete).
|
|
12
|
+
* @param args.promptTemplate - The prompt template to use (defaults to COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
13
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
14
|
+
*
|
|
15
|
+
* @returns An evaluator function that takes a {@link CompletenessEvaluationRecord} and returns a classification result
|
|
16
|
+
* indicating whether the conversation is `complete` or `incomplete`.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```ts
|
|
20
|
+
* const evaluator = createCompletenessEvaluator({ model: openai("gpt-4o-mini") });
|
|
21
|
+
* const result = await evaluator.evaluate({
|
|
22
|
+
* conversation:
|
|
23
|
+
* "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
|
|
24
|
+
* });
|
|
25
|
+
* console.log(result.label); // "incomplete"
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export function createCompletenessEvaluator(args) {
|
|
29
|
+
const { choices = COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
|
|
30
|
+
return createClassificationEvaluator({
|
|
31
|
+
...rest,
|
|
32
|
+
promptTemplate,
|
|
33
|
+
choices,
|
|
34
|
+
optimizationDirection,
|
|
35
|
+
name,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=createCompletenessEvaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createCompletenessEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createCompletenessEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4CAA4C,EAAE,MAAM,oCAAoC,CAAC;AAGlG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA0BhF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,2BAA2B,CAGzC,IAA2C;IAE3C,MAAM,EACJ,OAAO,GAAG,4CAA4C,CAAC,OAAO,EAC9D,cAAc,GAAG,4CAA4C,CAAC,QAAQ,EACtE,qBAAqB,GAAG,4CAA4C,CAAC,qBAAqB,EAC1F,IAAI,GAAG,4CAA4C,CAAC,IAAI,EACxD,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import type { CreateClassificationEvaluatorArgs } from "../types/evals.js";
|
|
2
2
|
import type { ClassificationEvaluator } from "./ClassificationEvaluator.js";
|
|
3
|
+
/**
|
|
4
|
+
* Arguments for the document relevance evaluator.
|
|
5
|
+
*
|
|
6
|
+
* @deprecated Use {@link RetrievalRelevanceEvaluatorArgs} instead. This type will be
|
|
7
|
+
* removed in the next major release.
|
|
8
|
+
*/
|
|
3
9
|
export interface DocumentRelevanceEvaluatorArgs<RecordType extends Record<string, unknown> = DocumentRelevanceEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
|
|
4
10
|
optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
|
|
5
11
|
name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
|
|
@@ -8,6 +14,9 @@ export interface DocumentRelevanceEvaluatorArgs<RecordType extends Record<string
|
|
|
8
14
|
}
|
|
9
15
|
/**
|
|
10
16
|
* A record to be evaluated by the document relevance evaluator.
|
|
17
|
+
*
|
|
18
|
+
* @deprecated Use {@link RetrievalRelevanceEvaluationRecord} instead. Rename
|
|
19
|
+
* `documentText` to `context`. This type will be removed in the next major release.
|
|
11
20
|
*/
|
|
12
21
|
export interface DocumentRelevanceEvaluationRecord {
|
|
13
22
|
input: string;
|
|
@@ -17,6 +26,12 @@ export interface DocumentRelevanceEvaluationRecord {
|
|
|
17
26
|
/**
|
|
18
27
|
* Creates a document relevance evaluator function.
|
|
19
28
|
*
|
|
29
|
+
* @deprecated Use {@link createRetrievalRelevanceEvaluator} instead. Rename the
|
|
30
|
+
* `documentText` input field to `context`, and update code that checks for the
|
|
31
|
+
* `unrelated` label to check for `irrelevant`. Pass one document as `context` to
|
|
32
|
+
* preserve per-document evaluation. This function will be removed in the next major
|
|
33
|
+
* release.
|
|
34
|
+
*
|
|
20
35
|
* This function returns an evaluator that determines whether a given document text
|
|
21
36
|
* is relevant to a provided input question. The evaluator uses a classification model
|
|
22
37
|
* and a prompt template to make its determination.
|
|
@@ -27,18 +42,8 @@ export interface DocumentRelevanceEvaluationRecord {
|
|
|
27
42
|
* @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCE_TEMPLATE).
|
|
28
43
|
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
29
44
|
*
|
|
30
|
-
* @returns An evaluator function that takes a {@link
|
|
45
|
+
* @returns An evaluator function that takes a {@link DocumentRelevanceEvaluationRecord} and returns a classification result
|
|
31
46
|
* indicating whether the document is relevant to the input question.
|
|
32
|
-
*
|
|
33
|
-
* @example
|
|
34
|
-
* ```ts
|
|
35
|
-
* const evaluator = createDocumentRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
36
|
-
* const result = await evaluator.evaluate({
|
|
37
|
-
* input: "What is the capital of France?",
|
|
38
|
-
* documentText: "Paris is the capital and most populous city of France.",
|
|
39
|
-
* });
|
|
40
|
-
* console.log(result.label); // "relevant" or "unrelated"
|
|
41
|
-
* ```
|
|
42
47
|
*/
|
|
43
48
|
export declare function createDocumentRelevanceEvaluator<RecordType extends Record<string, unknown> = DocumentRelevanceEvaluationRecord>(args: DocumentRelevanceEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
|
|
44
49
|
//# sourceMappingURL=createDocumentRelevanceEvaluator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createDocumentRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,8BAA8B,CAC7C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,iCAAiC,CACnC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED
|
|
1
|
+
{"version":3,"file":"createDocumentRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE;;;;;GAKG;AACH,MAAM,WAAW,8BAA8B,CAC7C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,iCAAiC,CACnC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;;;;GAKG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,iCAAiC,EAEnC,IAAI,EAAE,8BAA8B,CAAC,UAAU,CAAC,GAC/C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -3,6 +3,12 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
3
3
|
/**
|
|
4
4
|
* Creates a document relevance evaluator function.
|
|
5
5
|
*
|
|
6
|
+
* @deprecated Use {@link createRetrievalRelevanceEvaluator} instead. Rename the
|
|
7
|
+
* `documentText` input field to `context`, and update code that checks for the
|
|
8
|
+
* `unrelated` label to check for `irrelevant`. Pass one document as `context` to
|
|
9
|
+
* preserve per-document evaluation. This function will be removed in the next major
|
|
10
|
+
* release.
|
|
11
|
+
*
|
|
6
12
|
* This function returns an evaluator that determines whether a given document text
|
|
7
13
|
* is relevant to a provided input question. The evaluator uses a classification model
|
|
8
14
|
* and a prompt template to make its determination.
|
|
@@ -13,18 +19,8 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
13
19
|
* @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCE_TEMPLATE).
|
|
14
20
|
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
15
21
|
*
|
|
16
|
-
* @returns An evaluator function that takes a {@link
|
|
22
|
+
* @returns An evaluator function that takes a {@link DocumentRelevanceEvaluationRecord} and returns a classification result
|
|
17
23
|
* indicating whether the document is relevant to the input question.
|
|
18
|
-
*
|
|
19
|
-
* @example
|
|
20
|
-
* ```ts
|
|
21
|
-
* const evaluator = createDocumentRelevanceEvaluator({ model: openai("gpt-4o-mini") });
|
|
22
|
-
* const result = await evaluator.evaluate({
|
|
23
|
-
* input: "What is the capital of France?",
|
|
24
|
-
* documentText: "Paris is the capital and most populous city of France.",
|
|
25
|
-
* });
|
|
26
|
-
* console.log(result.label); // "relevant" or "unrelated"
|
|
27
|
-
* ```
|
|
28
24
|
*/
|
|
29
25
|
export function createDocumentRelevanceEvaluator(args) {
|
|
30
26
|
const { choices = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createDocumentRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kDAAkD,EAAE,MAAM,oCAAoC,CAAC;AAGxG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"createDocumentRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kDAAkD,EAAE,MAAM,oCAAoC,CAAC;AAGxG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAiChF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,gCAAgC,CAI9C,IAAgD;IAEhD,MAAM,EACJ,OAAO,GAAG,kDAAkD,CAAC,OAAO,EACpE,cAAc,GAAG,kDAAkD,CAAC,QAAQ,EAC5E,qBAAqB,GAAG,kDAAkD,CAAC,qBAAqB,EAChG,IAAI,GAAG,kDAAkD,CAAC,IAAI,EAC9D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { CreateClassificationEvaluatorArgs } from "../types/evals.js";
|
|
2
|
+
import type { ClassificationEvaluator } from "./ClassificationEvaluator.js";
|
|
3
|
+
export interface PiiDetectionEvaluatorArgs<RecordType extends Record<string, unknown> = PiiDetectionEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
|
|
4
|
+
optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
|
|
5
|
+
name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
|
|
6
|
+
choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
|
|
7
|
+
promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* A conversation record to be screened for personally identifiable information.
|
|
11
|
+
*/
|
|
12
|
+
export type PiiDetectionEvaluationRecord = {
|
|
13
|
+
/**
|
|
14
|
+
* The full conversation record, including system instructions, turns,
|
|
15
|
+
* tool calls, and retrieved content.
|
|
16
|
+
*/
|
|
17
|
+
conversation: string;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Creates a PII detection evaluator.
|
|
21
|
+
*
|
|
22
|
+
* This function returns an evaluator that screens a conversation record for
|
|
23
|
+
* personally identifiable information, including content in tool calls,
|
|
24
|
+
* tool results, and retrieved documents the end user may never have seen.
|
|
25
|
+
*
|
|
26
|
+
* @param args - The arguments for creating the PII detection evaluator.
|
|
27
|
+
* @param args.model - The model to use for classification.
|
|
28
|
+
* @param args.choices - The possible classification choices (defaults to pii_detected/no_pii_detected).
|
|
29
|
+
* @param args.promptTemplate - The prompt template to use (defaults to PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
30
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
31
|
+
*
|
|
32
|
+
* @returns An evaluator function that takes a {@link PiiDetectionEvaluationRecord} and returns a classification result
|
|
33
|
+
* indicating whether the record contains PII (`pii_detected`) or not (`no_pii_detected`).
|
|
34
|
+
*
|
|
35
|
+
* @example
|
|
36
|
+
* ```ts
|
|
37
|
+
* const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
|
|
38
|
+
* const result = await evaluator.evaluate({
|
|
39
|
+
* conversation:
|
|
40
|
+
* "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
|
|
41
|
+
* });
|
|
42
|
+
* console.log(result.label); // "pii_detected"
|
|
43
|
+
* ```
|
|
44
|
+
*/
|
|
45
|
+
export declare function createPiiDetectionEvaluator<RecordType extends Record<string, unknown> = PiiDetectionEvaluationRecord>(args: PiiDetectionEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
|
|
46
|
+
//# sourceMappingURL=createPiiDetectionEvaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createPiiDetectionEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createPiiDetectionEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,yBAAyB,CACxC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,4BAA4B,CACzE,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,MAAM,4BAA4B,GAAG;IACzC;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,2BAA2B,CACzC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,4BAA4B,EAEzE,IAAI,EAAE,yBAAyB,CAAC,UAAU,CAAC,GAC1C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
|
|
2
|
+
import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a PII detection evaluator.
|
|
5
|
+
*
|
|
6
|
+
* This function returns an evaluator that screens a conversation record for
|
|
7
|
+
* personally identifiable information, including content in tool calls,
|
|
8
|
+
* tool results, and retrieved documents the end user may never have seen.
|
|
9
|
+
*
|
|
10
|
+
* @param args - The arguments for creating the PII detection evaluator.
|
|
11
|
+
* @param args.model - The model to use for classification.
|
|
12
|
+
* @param args.choices - The possible classification choices (defaults to pii_detected/no_pii_detected).
|
|
13
|
+
* @param args.promptTemplate - The prompt template to use (defaults to PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template).
|
|
14
|
+
* @param args.telemetry - The telemetry to use for the evaluator.
|
|
15
|
+
*
|
|
16
|
+
* @returns An evaluator function that takes a {@link PiiDetectionEvaluationRecord} and returns a classification result
|
|
17
|
+
* indicating whether the record contains PII (`pii_detected`) or not (`no_pii_detected`).
|
|
18
|
+
*
|
|
19
|
+
* @example
|
|
20
|
+
* ```ts
|
|
21
|
+
* const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
|
|
22
|
+
* const result = await evaluator.evaluate({
|
|
23
|
+
* conversation:
|
|
24
|
+
* "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
|
|
25
|
+
* });
|
|
26
|
+
* console.log(result.label); // "pii_detected"
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
export function createPiiDetectionEvaluator(args) {
|
|
30
|
+
const { choices = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
|
|
31
|
+
return createClassificationEvaluator({
|
|
32
|
+
...rest,
|
|
33
|
+
promptTemplate,
|
|
34
|
+
choices,
|
|
35
|
+
optimizationDirection,
|
|
36
|
+
name,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=createPiiDetectionEvaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createPiiDetectionEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createPiiDetectionEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6CAA6C,EAAE,MAAM,oCAAoC,CAAC;AAGnG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAyBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,2BAA2B,CAGzC,IAA2C;IAE3C,MAAM,EACJ,OAAO,GAAG,6CAA6C,CAAC,OAAO,EAC/D,cAAc,GAAG,6CAA6C,CAAC,QAAQ,EACvE,qBAAqB,GAAG,6CAA6C,CAAC,qBAAqB,EAC3F,IAAI,GAAG,6CAA6C,CAAC,IAAI,EACzD,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -26,11 +26,10 @@ export interface RetrievalRelevanceEvaluationRecord {
|
|
|
26
26
|
*
|
|
27
27
|
* This function returns an evaluator that determines whether the external
|
|
28
28
|
* information retrieved during a step is relevant to the request it was
|
|
29
|
-
* serving.
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
* search, or content embedded in an LLM turn.
|
|
29
|
+
* serving. It is source-agnostic and scores the retrieved information as a
|
|
30
|
+
* whole, whether it came from a vector search, a tool or MCP call, a web search,
|
|
31
|
+
* or content embedded in an LLM turn. Pass one document as `context` to evaluate
|
|
32
|
+
* documents individually.
|
|
34
33
|
*
|
|
35
34
|
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
36
35
|
* @param args.model - The model to use for classification.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
|
|
@@ -5,11 +5,10 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
5
5
|
*
|
|
6
6
|
* This function returns an evaluator that determines whether the external
|
|
7
7
|
* information retrieved during a step is relevant to the request it was
|
|
8
|
-
* serving.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* search, or content embedded in an LLM turn.
|
|
8
|
+
* serving. It is source-agnostic and scores the retrieved information as a
|
|
9
|
+
* whole, whether it came from a vector search, a tool or MCP call, a web search,
|
|
10
|
+
* or content embedded in an LLM turn. Pass one document as `context` to evaluate
|
|
11
|
+
* documents individually.
|
|
13
12
|
*
|
|
14
13
|
* @param args - The arguments for creating the retrieval relevance evaluator.
|
|
15
14
|
* @param args.model - The model to use for classification.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF
|
|
1
|
+
{"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,UAAU,iCAAiC,CAI/C,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,mDAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,mDAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,mDAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,mDAAmD,CAAC,IAAI,EAC/D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
package/dist/esm/llm/index.d.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
export * from "./ClassificationEvaluator.js";
|
|
2
2
|
export * from "./createClassificationEvaluator.js";
|
|
3
3
|
export * from "./createClassifierFn.js";
|
|
4
|
+
export * from "./createCompletenessEvaluator.js";
|
|
4
5
|
export * from "./createConcisenessEvaluator.js";
|
|
5
6
|
export * from "./createCorrectnessEvaluator.js";
|
|
6
7
|
export * from "./createDocumentRelevanceEvaluator.js";
|
|
7
8
|
export * from "./createFaithfulnessEvaluator.js";
|
|
8
9
|
export * from "./createHallucinationEvaluator.js";
|
|
10
|
+
export * from "./createPiiDetectionEvaluator.js";
|
|
9
11
|
export * from "./createRefusalEvaluator.js";
|
|
10
12
|
export * from "./createRetrievalRelevanceEvaluator.js";
|
|
11
13
|
export * from "./createToolInvocationEvaluator.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
package/dist/esm/llm/index.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
export * from "./ClassificationEvaluator.js";
|
|
2
2
|
export * from "./createClassificationEvaluator.js";
|
|
3
3
|
export * from "./createClassifierFn.js";
|
|
4
|
+
export * from "./createCompletenessEvaluator.js";
|
|
4
5
|
export * from "./createConcisenessEvaluator.js";
|
|
5
6
|
export * from "./createCorrectnessEvaluator.js";
|
|
6
7
|
export * from "./createDocumentRelevanceEvaluator.js";
|
|
7
8
|
export * from "./createFaithfulnessEvaluator.js";
|
|
8
9
|
export * from "./createHallucinationEvaluator.js";
|
|
10
|
+
export * from "./createPiiDetectionEvaluator.js";
|
|
9
11
|
export * from "./createRefusalEvaluator.js";
|
|
10
12
|
export * from "./createRetrievalRelevanceEvaluator.js";
|
|
11
13
|
export * from "./createToolInvocationEvaluator.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
|