@arizeai/phoenix-evals 0.0.8 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arizeai/phoenix-evals",
-  "version": "0.0.8",
+  "version": "0.2.0",
   "description": "A library for running evaluations for AI use cases",
   "main": "dist/src/index.js",
   "module": "dist/esm/index.js",
@@ -50,7 +50,7 @@
     "typedoc": "^0.27.9",
     "typescript": "^5.8.2",
     "vitest": "^2.1.9",
-    "@arizeai/phoenix-client": "2.4.0"
+    "@arizeai/phoenix-client": "3.0.0"
   },
   "engines": {
     "node": ">=18"

package/src/default_templates/HALLUCINATION_TEMPLATE.ts

@@ -26,6 +26,6 @@ Is the answer above factual or hallucinated based on the query and reference tex
 `;
 
 export const HALLUCINATION_CHOICES = {
-  factual: 1,
-  hallucinated: 0,
+  hallucinated: 1,
+  factual: 0,
 };

package/src/llm/createClassificationEvaluator.ts

@@ -0,0 +1,13 @@
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
+import { createClassifierFn } from "./createClassifierFn";
+
+export function createClassificationEvaluator<
+  ExampleType extends Record<string, unknown>,
+>(args: CreateClassificationEvaluatorArgs): Evaluator<ExampleType> {
+  return {
+    name: args.name,
+    source: "LLM",
+    optimizationDirection: args.optimizationDirection,
+    evaluate: createClassifierFn(args),
+  };
+}

package/src/llm/{createClassifier.ts → createClassifierFn.ts}

@@ -22,9 +22,9 @@ function choicesToLabels(
 }
 
 /**
- * A function that serves as a factory that will output a classification evaluator
+ * A function that serves as a factory that will output a classification evaluator function
  */
-export function createClassifier<ExampleType extends Record<string, unknown>>(
+export function createClassifierFn<ExampleType extends Record<string, unknown>>(
   args: CreateClassifierArgs
 ): EvaluatorFn<ExampleType> {
   const { model, choices, promptTemplate, ...rest } = args;

package/src/llm/createDocumentRelevancyEvaluator.ts

@@ -1,14 +1,19 @@
-import { createClassifier } from "./createClassifier";
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
 import {
   DOCUMENT_RELEVANCY_TEMPLATE,
   DOCUMENT_RELEVANCY_CHOICES,
 } from "../default_templates/DOCUMENT_RELEVANCY_TEMPLATE";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface DocumentRelevancyEvaluatorArgs
-  extends Omit<CreateClassifierArgs, "promptTemplate" | "choices"> {
-  choices?: CreateClassifierArgs["choices"];
-  promptTemplate?: CreateClassifierArgs["promptTemplate"];
+  extends Omit<
+    CreateClassificationEvaluatorArgs,
+    "promptTemplate" | "choices" | "optimizationDirection" | "name"
+  > {
+  optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs["name"];
+  choices?: CreateClassificationEvaluatorArgs["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
 
 /**
@@ -38,7 +43,7 @@ export type DocumentRelevancyExample = {
  * @example
  * ```ts
  * const evaluator = createDocumentRelevancyEvaluator({ model: openai("gpt-4o-mini") });
- * const result = await evaluator({
+ * const result = await evaluator.evaluate({
  *   input: "What is the capital of France?",
  *   documentText: "Paris is the capital and most populous city of France.",
  * });
@@ -47,18 +52,21 @@ export type DocumentRelevancyExample = {
  */
 export function createDocumentRelevancyEvaluator(
   args: DocumentRelevancyEvaluatorArgs
-): EvaluatorFn<DocumentRelevancyExample> {
+): Evaluator<DocumentRelevancyExample> {
   const {
     choices = DOCUMENT_RELEVANCY_CHOICES,
     promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE,
+    optimizationDirection = "MAXIMIZE",
+    name = "document_relevancy",
     ...rest
   } = args;
-  const documentRelevancyEvaluatorFn =
-    createClassifier<DocumentRelevancyExample>({
-      ...args,
-      promptTemplate,
-      choices,
-      ...rest,
-    });
-  return documentRelevancyEvaluatorFn;
+
+  return createClassificationEvaluator<DocumentRelevancyExample>({
+    ...args,
+    promptTemplate,
+    choices,
+    optimizationDirection,
+    name,
+    ...rest,
+  });
 }

package/src/llm/createHallucinationEvaluator.ts

@@ -1,14 +1,19 @@
-import { createClassifier } from "./createClassifier";
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
+import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
 import {
   HALLUCINATION_TEMPLATE,
   HALLUCINATION_CHOICES,
 } from "../default_templates/HALLUCINATION_TEMPLATE";
+import { createClassificationEvaluator } from "./createClassificationEvaluator";
 
 export interface HallucinationEvaluatorArgs
-  extends Omit<CreateClassifierArgs, "promptTemplate" | "choices"> {
-  choices?: CreateClassifierArgs["choices"];
-  promptTemplate?: CreateClassifierArgs["promptTemplate"];
+  extends Omit<
+    CreateClassificationEvaluatorArgs,
+    "promptTemplate" | "choices" | "optimizationDirection" | "name"
+  > {
+  optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
+  name?: CreateClassificationEvaluatorArgs["name"];
+  choices?: CreateClassificationEvaluatorArgs["choices"];
+  promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
 }
 
 /**
@@ -28,17 +33,20 @@ export type HallucinationExample = {
  */
 export function createHallucinationEvaluator(
   args: HallucinationEvaluatorArgs
-): EvaluatorFn<HallucinationExample> {
+): Evaluator<HallucinationExample> {
   const {
     choices = HALLUCINATION_CHOICES,
     promptTemplate = HALLUCINATION_TEMPLATE,
+    optimizationDirection = "MINIMIZE",
+    name = "hallucination",
     ...rest
   } = args;
-  const hallucinationEvaluatorFn = createClassifier<HallucinationExample>({
+  return createClassificationEvaluator<HallucinationExample>({
     ...args,
     promptTemplate,
     choices,
+    optimizationDirection,
+    name,
     ...rest,
   });
-  return hallucinationEvaluatorFn;
 }

package/src/llm/index.ts

@@ -1,4 +1,5 @@
 export * from "./generateClassification";
-export * from "./createClassifier";
+export * from "./createClassifierFn";
+export * from "./createClassificationEvaluator";
 export * from "./createHallucinationEvaluator";
 export * from "./createDocumentRelevancyEvaluator";

package/src/types/evals.ts

@@ -78,6 +78,55 @@ export interface CreateClassifierArgs extends WithTelemetry {
   promptTemplate: string;
 }
 
+export interface CreateClassificationEvaluatorArgs
+  extends CreateClassifierArgs {
+  /**
+   * The name of the metric that the evaluator produces
+   * E.x. "correctness"
+   */
+  name: string;
+  /**
+   * If present, represents the direction in which you want the metric to be optimized
+   * E.x. "MAXIMIZE" means you want the number to be higher.
+   */
+  optimizationDirection?: OptimizationDirection;
+}
+
 export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (
   args: ExampleType
 ) => Promise<EvaluationResult>;
+
+/**
+ * The source of the evaluation
+ */
+type EvaluationSource = "LLM" | "CODE";
+
+/**
+ * The direction to optimize the numeric evaluation score
+ * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
+ */
+type OptimizationDirection = "MAXIMIZE" | "MINIMIZE";
+
+/**
+ * The Base Evaluator interface
+ * This is the interface that all evaluators must implement
+ */
+export interface Evaluator<ExampleType extends Record<string, unknown>> {
+  /**
+   * The name of the evaluator / the metric that it measures
+   */
+  name: string;
+  /**
+   * The source of the evaluation. Also known as the "kind" of evaluator.
+   */
+  source: EvaluationSource;
+  /**
+   * The direction to optimize the numeric evaluation score
+   * E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
+   */
+  optimizationDirection?: OptimizationDirection;
+  /**
+   * The function that evaluates the example
+   */
+  evaluate: EvaluatorFn<ExampleType>;
+}

package/dist/esm/llm/createClassifier.d.ts

@@ -1,6 +0,0 @@
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals.js";
-/**
- * A function that serves as a factory that will output a classification evaluator
- */
-export declare function createClassifier<ExampleType extends Record<string, unknown>>(args: CreateClassifierArgs): EvaluatorFn<ExampleType>;
-//# sourceMappingURL=createClassifier.d.ts.map

package/dist/esm/llm/createClassifier.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"createClassifier.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifier.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC1E,IAAI,EAAE,oBAAoB,GACzB,WAAW,CAAC,WAAW,CAAC,CA4B1B"}

package/dist/esm/llm/createClassifier.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"createClassifier.js","sourceRoot":"","sources":["../../../src/llm/createClassifier.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,IAA0B;IAE1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;IAEzD,OAAO,KAAK,EAAE,IAAiB,EAA6B,EAAE;QAC5D,MAAM,iBAAiB,GAAG;YACxB,GAAG,IAAI;SACR,CAAC;QAEF,MAAM,MAAM,GAAG,cAAc,CAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC;YAClD,KAAK;YACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC;YAChC,MAAM;YACN,GAAG,IAAI;SACR,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,OAAO;YACL,KAAK;YACL,GAAG,cAAc;SAClB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/llm/createClassifier.d.ts

@@ -1,6 +0,0 @@
-import { CreateClassifierArgs, EvaluatorFn } from "../types/evals";
-/**
- * A function that serves as a factory that will output a classification evaluator
- */
-export declare function createClassifier<ExampleType extends Record<string, unknown>>(args: CreateClassifierArgs): EvaluatorFn<ExampleType>;
-//# sourceMappingURL=createClassifier.d.ts.map

package/dist/src/llm/createClassifier.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"createClassifier.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifier.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,oBAAoB,EACpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC1E,IAAI,EAAE,oBAAoB,GACzB,WAAW,CAAC,WAAW,CAAC,CA4B1B"}

package/dist/src/llm/createClassifier.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"createClassifier.js","sourceRoot":"","sources":["../../../src/llm/createClassifier.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA0BA,4CA8BC;AAlDD,qEAAkE;AAClE,0CAA6C;AAE7C;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAgB,gBAAgB,CAC9B,IAA0B;IAE1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAAlD,sCAA2C,CAAO,CAAC;IAEzD,OAAO,KAAK,EAAE,IAAiB,EAA6B,EAAE;QAC5D,MAAM,iBAAiB,qBAClB,IAAI,CACR,CAAC;QAEF,MAAM,MAAM,GAAG,IAAA,yBAAc,EAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,IAAA,+CAAsB,kBACjD,KAAK,EACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,EAChC,MAAM,IACH,IAAI,EACP,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,uBACE,KAAK,IACF,cAAc,EACjB;IACJ,CAAC,CAAC;AACJ,CAAC"}