@arizeai/phoenix-evals 2.3.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +256 -0
  2. package/README.md +7 -5
  3. package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
  4. package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
  5. package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js +93 -0
  6. package/dist/esm/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
  7. package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
  8. package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
  9. package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js +119 -0
  10. package/dist/esm/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
  11. package/dist/esm/__generated__/default_templates/index.d.ts +2 -0
  12. package/dist/esm/__generated__/default_templates/index.d.ts.map +1 -1
  13. package/dist/esm/__generated__/default_templates/index.js +2 -0
  14. package/dist/esm/__generated__/default_templates/index.js.map +1 -1
  15. package/dist/esm/llm/createCompletenessEvaluator.d.ts +46 -0
  16. package/dist/esm/llm/createCompletenessEvaluator.d.ts.map +1 -0
  17. package/dist/esm/llm/createCompletenessEvaluator.js +38 -0
  18. package/dist/esm/llm/createCompletenessEvaluator.js.map +1 -0
  19. package/dist/esm/llm/createDocumentRelevanceEvaluator.d.ts +16 -11
  20. package/dist/esm/llm/createDocumentRelevanceEvaluator.d.ts.map +1 -1
  21. package/dist/esm/llm/createDocumentRelevanceEvaluator.js +7 -11
  22. package/dist/esm/llm/createDocumentRelevanceEvaluator.js.map +1 -1
  23. package/dist/esm/llm/createPiiDetectionEvaluator.d.ts +46 -0
  24. package/dist/esm/llm/createPiiDetectionEvaluator.d.ts.map +1 -0
  25. package/dist/esm/llm/createPiiDetectionEvaluator.js +39 -0
  26. package/dist/esm/llm/createPiiDetectionEvaluator.js.map +1 -0
  27. package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +4 -5
  28. package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -1
  29. package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +4 -5
  30. package/dist/esm/llm/createRetrievalRelevanceEvaluator.js.map +1 -1
  31. package/dist/esm/llm/index.d.ts +2 -0
  32. package/dist/esm/llm/index.d.ts.map +1 -1
  33. package/dist/esm/llm/index.js +2 -0
  34. package/dist/esm/llm/index.js.map +1 -1
  35. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  36. package/dist/esm/utils/bindEvaluator.d.ts +4 -5
  37. package/dist/esm/utils/bindEvaluator.d.ts.map +1 -1
  38. package/dist/esm/utils/bindEvaluator.js +4 -5
  39. package/dist/esm/utils/bindEvaluator.js.map +1 -1
  40. package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
  41. package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
  42. package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js +96 -0
  43. package/dist/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
  44. package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts +3 -0
  45. package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map +1 -0
  46. package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js +122 -0
  47. package/dist/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.js.map +1 -0
  48. package/dist/src/__generated__/default_templates/index.d.ts +2 -0
  49. package/dist/src/__generated__/default_templates/index.d.ts.map +1 -1
  50. package/dist/src/__generated__/default_templates/index.js +5 -1
  51. package/dist/src/__generated__/default_templates/index.js.map +1 -1
  52. package/dist/src/llm/createCompletenessEvaluator.d.ts +46 -0
  53. package/dist/src/llm/createCompletenessEvaluator.d.ts.map +1 -0
  54. package/dist/src/llm/createCompletenessEvaluator.js +49 -0
  55. package/dist/src/llm/createCompletenessEvaluator.js.map +1 -0
  56. package/dist/src/llm/createDocumentRelevanceEvaluator.d.ts +16 -11
  57. package/dist/src/llm/createDocumentRelevanceEvaluator.d.ts.map +1 -1
  58. package/dist/src/llm/createDocumentRelevanceEvaluator.js +7 -11
  59. package/dist/src/llm/createDocumentRelevanceEvaluator.js.map +1 -1
  60. package/dist/src/llm/createPiiDetectionEvaluator.d.ts +46 -0
  61. package/dist/src/llm/createPiiDetectionEvaluator.d.ts.map +1 -0
  62. package/dist/src/llm/createPiiDetectionEvaluator.js +50 -0
  63. package/dist/src/llm/createPiiDetectionEvaluator.js.map +1 -0
  64. package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +4 -5
  65. package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts.map +1 -1
  66. package/dist/src/llm/createRetrievalRelevanceEvaluator.js +4 -5
  67. package/dist/src/llm/createRetrievalRelevanceEvaluator.js.map +1 -1
  68. package/dist/src/llm/index.d.ts +2 -0
  69. package/dist/src/llm/index.d.ts.map +1 -1
  70. package/dist/src/llm/index.js +2 -0
  71. package/dist/src/llm/index.js.map +1 -1
  72. package/dist/src/utils/bindEvaluator.d.ts +4 -5
  73. package/dist/src/utils/bindEvaluator.d.ts.map +1 -1
  74. package/dist/src/utils/bindEvaluator.js +4 -5
  75. package/dist/src/utils/bindEvaluator.js.map +1 -1
  76. package/dist/tsconfig.tsbuildinfo +1 -1
  77. package/docs/llm-evaluators.mdx +1 -1
  78. package/package.json +12 -12
  79. package/src/__generated__/default_templates/COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.ts +95 -0
  80. package/src/__generated__/default_templates/PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.ts +121 -0
  81. package/src/__generated__/default_templates/index.ts +2 -0
  82. package/src/llm/createCompletenessEvaluator.ts +74 -0
  83. package/src/llm/createDocumentRelevanceEvaluator.ts +16 -11
  84. package/src/llm/createPiiDetectionEvaluator.ts +74 -0
  85. package/src/llm/createRetrievalRelevanceEvaluator.ts +4 -5
  86. package/src/llm/index.ts +2 -0
  87. package/src/utils/bindEvaluator.ts +4 -5
@@ -0,0 +1,38 @@
1
+ import { COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
2
+ import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
3
+ /**
4
+ * Creates a completeness evaluator.
5
+ *
6
+ * This function returns an evaluator that checks whether every active user
7
+ * request in a conversation was actually completed, not merely acknowledged.
8
+ *
9
+ * @param args - The arguments for creating the completeness evaluator.
10
+ * @param args.model - The model to use for classification.
11
+ * @param args.choices - The possible classification choices (defaults to complete/incomplete).
12
+ * @param args.promptTemplate - The prompt template to use (defaults to COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.template).
13
+ * @param args.telemetry - The telemetry to use for the evaluator.
14
+ *
15
+ * @returns An evaluator function that takes a {@link CompletenessEvaluationRecord} and returns a classification result
16
+ * indicating whether the conversation is `complete` or `incomplete`.
17
+ *
18
+ * @example
19
+ * ```ts
20
+ * const evaluator = createCompletenessEvaluator({ model: openai("gpt-4o-mini") });
21
+ * const result = await evaluator.evaluate({
22
+ * conversation:
23
+ * "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
24
+ * });
25
+ * console.log(result.label); // "incomplete"
26
+ * ```
27
+ */
28
+ export function createCompletenessEvaluator(args) {
29
+ const { choices = COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = COMPLETENESS_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
30
+ return createClassificationEvaluator({
31
+ ...rest,
32
+ promptTemplate,
33
+ choices,
34
+ optimizationDirection,
35
+ name,
36
+ });
37
+ }
38
+ //# sourceMappingURL=createCompletenessEvaluator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createCompletenessEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createCompletenessEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4CAA4C,EAAE,MAAM,oCAAoC,CAAC;AAGlG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA0BhF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,2BAA2B,CAGzC,IAA2C;IAE3C,MAAM,EACJ,OAAO,GAAG,4CAA4C,CAAC,OAAO,EAC9D,cAAc,GAAG,4CAA4C,CAAC,QAAQ,EACtE,qBAAqB,GAAG,4CAA4C,CAAC,qBAAqB,EAC1F,IAAI,GAAG,4CAA4C,CAAC,IAAI,EACxD,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
@@ -1,5 +1,11 @@
1
1
  import type { CreateClassificationEvaluatorArgs } from "../types/evals.js";
2
2
  import type { ClassificationEvaluator } from "./ClassificationEvaluator.js";
3
+ /**
4
+ * Arguments for the document relevance evaluator.
5
+ *
6
+ * @deprecated Use {@link RetrievalRelevanceEvaluatorArgs} instead. This type will be
7
+ * removed in the next major release.
8
+ */
3
9
  export interface DocumentRelevanceEvaluatorArgs<RecordType extends Record<string, unknown> = DocumentRelevanceEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
4
10
  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
5
11
  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
@@ -8,6 +14,9 @@ export interface DocumentRelevanceEvaluatorArgs<RecordType extends Record<string
8
14
  }
9
15
  /**
10
16
  * A record to be evaluated by the document relevance evaluator.
17
+ *
18
+ * @deprecated Use {@link RetrievalRelevanceEvaluationRecord} instead. Rename
19
+ * `documentText` to `context`. This type will be removed in the next major release.
11
20
  */
12
21
  export interface DocumentRelevanceEvaluationRecord {
13
22
  input: string;
@@ -17,6 +26,12 @@ export interface DocumentRelevanceEvaluationRecord {
17
26
  /**
18
27
  * Creates a document relevance evaluator function.
19
28
  *
29
+ * @deprecated Use {@link createRetrievalRelevanceEvaluator} instead. Rename the
30
+ * `documentText` input field to `context`, and update code that checks for the
31
+ * `unrelated` label to check for `irrelevant`. Pass one document as `context` to
32
+ * preserve per-document evaluation. This function will be removed in the next major
33
+ * release.
34
+ *
20
35
  * This function returns an evaluator that determines whether a given document text
21
36
  * is relevant to a provided input question. The evaluator uses a classification model
22
37
  * and a prompt template to make its determination.
@@ -27,18 +42,8 @@ export interface DocumentRelevanceEvaluationRecord {
27
42
  * @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCE_TEMPLATE).
28
43
  * @param args.telemetry - The telemetry to use for the evaluator.
29
44
  *
30
- * @returns An evaluator function that takes a {@link DocumentRelevanceExample} and returns a classification result
45
+ * @returns An evaluator function that takes a {@link DocumentRelevanceEvaluationRecord} and returns a classification result
31
46
  * indicating whether the document is relevant to the input question.
32
- *
33
- * @example
34
- * ```ts
35
- * const evaluator = createDocumentRelevanceEvaluator({ model: openai("gpt-4o-mini") });
36
- * const result = await evaluator.evaluate({
37
- * input: "What is the capital of France?",
38
- * documentText: "Paris is the capital and most populous city of France.",
39
- * });
40
- * console.log(result.label); // "relevant" or "unrelated"
41
- * ```
42
47
  */
43
48
  export declare function createDocumentRelevanceEvaluator<RecordType extends Record<string, unknown> = DocumentRelevanceEvaluationRecord>(args: DocumentRelevanceEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
44
49
  //# sourceMappingURL=createDocumentRelevanceEvaluator.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"createDocumentRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,8BAA8B,CAC7C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,iCAAiC,CACnC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,iCAAiC,EAEnC,IAAI,EAAE,8BAA8B,CAAC,UAAU,CAAC,GAC/C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
1
+ {"version":3,"file":"createDocumentRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE;;;;;GAKG;AACH,MAAM,WAAW,8BAA8B,CAC7C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,iCAAiC,CACnC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;;;;GAKG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,iCAAiC,EAEnC,IAAI,EAAE,8BAA8B,CAAC,UAAU,CAAC,GAC/C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
@@ -3,6 +3,12 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
3
3
  /**
4
4
  * Creates a document relevance evaluator function.
5
5
  *
6
+ * @deprecated Use {@link createRetrievalRelevanceEvaluator} instead. Rename the
7
+ * `documentText` input field to `context`, and update code that checks for the
8
+ * `unrelated` label to check for `irrelevant`. Pass one document as `context` to
9
+ * preserve per-document evaluation. This function will be removed in the next major
10
+ * release.
11
+ *
6
12
  * This function returns an evaluator that determines whether a given document text
7
13
  * is relevant to a provided input question. The evaluator uses a classification model
8
14
  * and a prompt template to make its determination.
@@ -13,18 +19,8 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
13
19
  * @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCE_TEMPLATE).
14
20
  * @param args.telemetry - The telemetry to use for the evaluator.
15
21
  *
16
- * @returns An evaluator function that takes a {@link DocumentRelevanceExample} and returns a classification result
22
+ * @returns An evaluator function that takes a {@link DocumentRelevanceEvaluationRecord} and returns a classification result
17
23
  * indicating whether the document is relevant to the input question.
18
- *
19
- * @example
20
- * ```ts
21
- * const evaluator = createDocumentRelevanceEvaluator({ model: openai("gpt-4o-mini") });
22
- * const result = await evaluator.evaluate({
23
- * input: "What is the capital of France?",
24
- * documentText: "Paris is the capital and most populous city of France.",
25
- * });
26
- * console.log(result.label); // "relevant" or "unrelated"
27
- * ```
28
24
  */
29
25
  export function createDocumentRelevanceEvaluator(args) {
30
26
  const { choices = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
@@ -1 +1 @@
1
- {"version":3,"file":"createDocumentRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kDAAkD,EAAE,MAAM,oCAAoC,CAAC;AAGxG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAwBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,gCAAgC,CAI9C,IAAgD;IAEhD,MAAM,EACJ,OAAO,GAAG,kDAAkD,CAAC,OAAO,EACpE,cAAc,GAAG,kDAAkD,CAAC,QAAQ,EAC5E,qBAAqB,GAAG,kDAAkD,CAAC,qBAAqB,EAChG,IAAI,GAAG,kDAAkD,CAAC,IAAI,EAC9D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"createDocumentRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kDAAkD,EAAE,MAAM,oCAAoC,CAAC;AAGxG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAiChF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,gCAAgC,CAI9C,IAAgD;IAEhD,MAAM,EACJ,OAAO,GAAG,kDAAkD,CAAC,OAAO,EACpE,cAAc,GAAG,kDAAkD,CAAC,QAAQ,EAC5E,qBAAqB,GAAG,kDAAkD,CAAC,qBAAqB,EAChG,IAAI,GAAG,kDAAkD,CAAC,IAAI,EAC9D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,46 @@
1
+ import type { CreateClassificationEvaluatorArgs } from "../types/evals.js";
2
+ import type { ClassificationEvaluator } from "./ClassificationEvaluator.js";
3
+ export interface PiiDetectionEvaluatorArgs<RecordType extends Record<string, unknown> = PiiDetectionEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
4
+ optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
5
+ name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
6
+ choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
7
+ promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
8
+ }
9
+ /**
10
+ * A conversation record to be screened for personally identifiable information.
11
+ */
12
+ export type PiiDetectionEvaluationRecord = {
13
+ /**
14
+ * The full conversation record, including system instructions, turns,
15
+ * tool calls, and retrieved content.
16
+ */
17
+ conversation: string;
18
+ };
19
+ /**
20
+ * Creates a PII detection evaluator.
21
+ *
22
+ * This function returns an evaluator that screens a conversation record for
23
+ * personally identifiable information, including content in tool calls,
24
+ * tool results, and retrieved documents the end user may never have seen.
25
+ *
26
+ * @param args - The arguments for creating the PII detection evaluator.
27
+ * @param args.model - The model to use for classification.
28
+ * @param args.choices - The possible classification choices (defaults to pii_detected/no_pii_detected).
29
+ * @param args.promptTemplate - The prompt template to use (defaults to PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template).
30
+ * @param args.telemetry - The telemetry to use for the evaluator.
31
+ *
32
+ * @returns An evaluator function that takes a {@link PiiDetectionEvaluationRecord} and returns a classification result
33
+ * indicating whether the record contains PII (`pii_detected`) or not (`no_pii_detected`).
34
+ *
35
+ * @example
36
+ * ```ts
37
+ * const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
38
+ * const result = await evaluator.evaluate({
39
+ * conversation:
40
+ * "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
41
+ * });
42
+ * console.log(result.label); // "pii_detected"
43
+ * ```
44
+ */
45
+ export declare function createPiiDetectionEvaluator<RecordType extends Record<string, unknown> = PiiDetectionEvaluationRecord>(args: PiiDetectionEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
46
+ //# sourceMappingURL=createPiiDetectionEvaluator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createPiiDetectionEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createPiiDetectionEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,yBAAyB,CACxC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,4BAA4B,CACzE,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,MAAM,4BAA4B,GAAG;IACzC;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,2BAA2B,CACzC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,4BAA4B,EAEzE,IAAI,EAAE,yBAAyB,CAAC,UAAU,CAAC,GAC1C,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
@@ -0,0 +1,39 @@
1
+ import { PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates/index.js";
2
+ import { createClassificationEvaluator } from "./createClassificationEvaluator.js";
3
+ /**
4
+ * Creates a PII detection evaluator.
5
+ *
6
+ * This function returns an evaluator that screens a conversation record for
7
+ * personally identifiable information, including content in tool calls,
8
+ * tool results, and retrieved documents the end user may never have seen.
9
+ *
10
+ * @param args - The arguments for creating the PII detection evaluator.
11
+ * @param args.model - The model to use for classification.
12
+ * @param args.choices - The possible classification choices (defaults to pii_detected/no_pii_detected).
13
+ * @param args.promptTemplate - The prompt template to use (defaults to PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template).
14
+ * @param args.telemetry - The telemetry to use for the evaluator.
15
+ *
16
+ * @returns An evaluator function that takes a {@link PiiDetectionEvaluationRecord} and returns a classification result
17
+ * indicating whether the record contains PII (`pii_detected`) or not (`no_pii_detected`).
18
+ *
19
+ * @example
20
+ * ```ts
21
+ * const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
22
+ * const result = await evaluator.evaluate({
23
+ * conversation:
24
+ * "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
25
+ * });
26
+ * console.log(result.label); // "pii_detected"
27
+ * ```
28
+ */
29
+ export function createPiiDetectionEvaluator(args) {
30
+ const { choices = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.choices, promptTemplate = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.template, optimizationDirection = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection, name = PII_DETECTION_CLASSIFICATION_EVALUATOR_CONFIG.name, ...rest } = args;
31
+ return createClassificationEvaluator({
32
+ ...rest,
33
+ promptTemplate,
34
+ choices,
35
+ optimizationDirection,
36
+ name,
37
+ });
38
+ }
39
+ //# sourceMappingURL=createPiiDetectionEvaluator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createPiiDetectionEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createPiiDetectionEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6CAA6C,EAAE,MAAM,oCAAoC,CAAC;AAGnG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAyBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,2BAA2B,CAGzC,IAA2C;IAE3C,MAAM,EACJ,OAAO,GAAG,6CAA6C,CAAC,OAAO,EAC/D,cAAc,GAAG,6CAA6C,CAAC,QAAQ,EACvE,qBAAqB,GAAG,6CAA6C,CAAC,qBAAqB,EAC3F,IAAI,GAAG,6CAA6C,CAAC,IAAI,EACzD,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
@@ -26,11 +26,10 @@ export interface RetrievalRelevanceEvaluationRecord {
26
26
  *
27
27
  * This function returns an evaluator that determines whether the external
28
28
  * information retrieved during a step is relevant to the request it was
29
- * serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
30
- * single document against a question, this evaluator is source-agnostic and
31
- * scores the retrieved information as a whole (holistically, per retrieval
32
- * step), whether it came from a vector search, a tool or MCP call, a web
33
- * search, or content embedded in an LLM turn.
29
+ * serving. It is source-agnostic and scores the retrieved information as a
30
+ * whole, whether it came from a vector search, a tool or MCP call, a web search,
31
+ * or content embedded in an LLM turn. Pass one document as `context` to evaluate
32
+ * documents individually.
34
33
  *
35
34
  * @param args - The arguments for creating the retrieval relevance evaluator.
36
35
  * @param args.model - The model to use for classification.
@@ -1 +1 @@
1
- {"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
1
+ {"version":3,"file":"createRetrievalRelevanceEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGzE,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,CACpC,SAAQ,IAAI,CACZ,iCAAiC,CAAC,UAAU,CAAC,EAC7C,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACC,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,uBAAuB,CAAC,CAAC;IAC/F,IAAI,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE,cAAc,CAAC,EAAE,iCAAiC,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,oEAAoE;AACpE,MAAM,WAAW,kCAAkC;IACjD;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,iCAAiC,CAC/C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACxC,kCAAkC,EAEpC,IAAI,EAAE,+BAA+B,CAAC,UAAU,CAAC,GAChD,uBAAuB,CAAC,UAAU,CAAC,CAerC"}
@@ -5,11 +5,10 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
5
5
  *
6
6
  * This function returns an evaluator that determines whether the external
7
7
  * information retrieved during a step is relevant to the request it was
8
- * serving. Unlike {@link createDocumentRelevanceEvaluator}, which judges a
9
- * single document against a question, this evaluator is source-agnostic and
10
- * scores the retrieved information as a whole (holistically, per retrieval
11
- * step), whether it came from a vector search, a tool or MCP call, a web
12
- * search, or content embedded in an LLM turn.
8
+ * serving. It is source-agnostic and scores the retrieved information as a
9
+ * whole, whether it came from a vector search, a tool or MCP call, a web search,
10
+ * or content embedded in an LLM turn. Pass one document as `context` to evaluate
11
+ * documents individually.
13
12
  *
14
13
  * @param args - The arguments for creating the retrieval relevance evaluator.
15
14
  * @param args.model - The model to use for classification.
@@ -1 +1 @@
1
- {"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,UAAU,iCAAiC,CAI/C,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,mDAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,mDAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,mDAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,mDAAmD,CAAC,IAAI,EAC/D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"createRetrievalRelevanceEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createRetrievalRelevanceEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mDAAmD,EAAE,MAAM,oCAAoC,CAAC;AAGzG,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AA+BhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,UAAU,iCAAiC,CAI/C,IAAiD;IAEjD,MAAM,EACJ,OAAO,GAAG,mDAAmD,CAAC,OAAO,EACrE,cAAc,GAAG,mDAAmD,CAAC,QAAQ,EAC7E,qBAAqB,GAAG,mDAAmD,CAAC,qBAAqB,EACjG,IAAI,GAAG,mDAAmD,CAAC,IAAI,EAC/D,GAAG,IAAI,EACR,GAAG,IAAI,CAAC;IACT,OAAO,6BAA6B,CAAa;QAC/C,GAAG,IAAI;QACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
@@ -1,11 +1,13 @@
1
1
  export * from "./ClassificationEvaluator.js";
2
2
  export * from "./createClassificationEvaluator.js";
3
3
  export * from "./createClassifierFn.js";
4
+ export * from "./createCompletenessEvaluator.js";
4
5
  export * from "./createConcisenessEvaluator.js";
5
6
  export * from "./createCorrectnessEvaluator.js";
6
7
  export * from "./createDocumentRelevanceEvaluator.js";
7
8
  export * from "./createFaithfulnessEvaluator.js";
8
9
  export * from "./createHallucinationEvaluator.js";
10
+ export * from "./createPiiDetectionEvaluator.js";
9
11
  export * from "./createRefusalEvaluator.js";
10
12
  export * from "./createRetrievalRelevanceEvaluator.js";
11
13
  export * from "./createToolInvocationEvaluator.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
@@ -1,11 +1,13 @@
1
1
  export * from "./ClassificationEvaluator.js";
2
2
  export * from "./createClassificationEvaluator.js";
3
3
  export * from "./createClassifierFn.js";
4
+ export * from "./createCompletenessEvaluator.js";
4
5
  export * from "./createConcisenessEvaluator.js";
5
6
  export * from "./createCorrectnessEvaluator.js";
6
7
  export * from "./createDocumentRelevanceEvaluator.js";
7
8
  export * from "./createFaithfulnessEvaluator.js";
8
9
  export * from "./createHallucinationEvaluator.js";
10
+ export * from "./createPiiDetectionEvaluator.js";
9
11
  export * from "./createRefusalEvaluator.js";
10
12
  export * from "./createRetrievalRelevanceEvaluator.js";
11
13
  export * from "./createToolInvocationEvaluator.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iCAAiC,CAAC;AAChD,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AACtD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,gBAAgB,CAAC"}