@arizeai/phoenix-evals 0.0.1 → 0.0.3

package/src/llm/generateClassification.ts

@@ -0,0 +1,42 @@
+import { ClassificationResult, WithLLM } from "../types/evals";
+import type { WithPrompt } from "../types/prompts";
+import { generateObject } from "ai";
+import { z } from "zod";
+
+interface ClassifyArgs extends WithLLM, WithPrompt {
+  /**
+   * The labels to classify the example into. E.x. ["correct", "incorrect"]
+   */
+  labels: [string, ...string[]];
+  /**
+   * The name of the schema for generating the label and explanation.
+   */
+  schemaName?: string;
+  /**
+   * The description of the schema for generating the label and explanation.
+   */
+  schemaDescription?: string;
+}
+/**
+ * A function that leverages an llm to perform a classification
+ */
+export async function generateClassification(
+  args: ClassifyArgs
+): Promise<ClassificationResult> {
+  const { labels, model, schemaName, schemaDescription, ...prompt } = args;
+
+  const result = await generateObject({
+    model,
+    schemaName,
+    schemaDescription,
+    schema: z.object({
+      explanation: z.string(), // We place the explanation in hopes it uses reasoning to explain the label.
+      label: z.enum(labels),
+    }),
+    ...prompt,
+  });
+  return {
+    label: result.object.label,
+    explanation: result.object.explanation,
+  };
+}

package/src/llm/index.ts

@@ -0,0 +1,3 @@
+export * from "./generateClassification";
+export * from "./createClassifier";
+export * from "./createHallucinationEvaluator";

package/src/template/applyTemplate.ts

@@ -0,0 +1,14 @@
+import { Template } from "../types/templating";
+import Mustache from "mustache";
+
+/**
+ * A function that applies a set of variables to a template (e.g. a prompt)
+ * Uses the Mustache library to apply the variables to the template
+ */
+export function formatTemplate(args: {
+  template: Template;
+  variables: Record<string, unknown>;
+}) {
+  const { template, variables } = args;
+  return Mustache.render(template, variables);
+}

package/src/template/index.ts

@@ -0,0 +1 @@
+export * from "./applyTemplate";

package/src/types/evals.ts

@@ -0,0 +1,84 @@
+import { LanguageModel } from "ai";
+
+/**
+ * The arguments for an evaluation
+ */
+export interface EvaluationArgs<OutputType, InputType> {
+  output: OutputType;
+  expected?: OutputType;
+  input?: InputType;
+  [key: string]: unknown;
+}
+
+export interface WithLLM {
+  model: LanguageModel;
+}
+
+export interface LLMEvaluationArgs<OutputType, InputType>
+  extends EvaluationArgs<OutputType, InputType>,
+    WithLLM {}
+
+/**
+ * The result of an evaluation
+ */
+export interface EvaluationResult {
+  /**
+   * The score of the evaluation.
+   * @example 0.95
+   */
+  score?: number;
+  /**
+   * The label of the evaluation.
+   * @example "correct"
+   */
+  label?: string;
+  /**
+   * The explanation of the evaluation.
+   * @example "The model correctly identified the sentiment of the text."
+   */
+  explanation?: string;
+}
+
+/**
+ * The result of a classification
+ */
+export interface ClassificationResult {
+  label: string;
+  explanation?: string;
+}
+
+/**
+ * The choice (e.g. the label and score mapping) of a classification based evaluation
+ */
+export interface ClassificationChoice {
+  label: string;
+  score: number;
+}
+
+/**
+ * A mapping of labels to scores
+ */
+export type ClassificationChoicesMap = Record<string, number>;
+
+/**
+ * The arguments for creating a classification-based evaluator
+ */
+export interface CreateClassifierArgs {
+  /**
+   * The LLM to use for classification / evaluation
+   */
+  model: LanguageModel;
+  /**
+   * The choices to classify the example into.
+   * e.g. { "correct": 1, "incorrect": 0 }
+   */
+  choices: ClassificationChoicesMap;
+  /**
+   * The prompt template to use for classification
+   */
+  promptTemplate: string;
+}
+
+export type EvaluatorFn<OutputType, InputType> = (
+  args: EvaluationArgs<OutputType, InputType>
+) => Promise<EvaluationResult>;

package/src/types/index.ts

@@ -0,0 +1,2 @@
+export * from "./evals";
+export * from "./templating";

package/src/types/prompts.ts

@@ -0,0 +1,21 @@
+import { CoreMessage, Message } from "ai";
+
+/**
+ * Prompt part of the AI function options.
+ * It contains a system message, a simple text prompt, or a list of messages.
+ * Note: this is pulled from the `ai` package and is used as a compatibility type.
+ */
+export interface WithPrompt {
+  /**
+   * System message to include in the prompt. Can be used with `prompt` or `messages`.
+   */
+  system?: string;
+  /**
+   * A simple text prompt. You can either use `prompt` or `messages` but not both.
+   */
+  prompt?: string;
+  /**
+   * A list of messages. You can either use `prompt` or `messages` but not both.
+   */
+  messages?: Array<CoreMessage> | Array<Omit<Message, "id">>;
+}

package/src/types/templating.ts

@@ -0,0 +1,2 @@
+export type Template = string;
+export type TemplateVariables = Record<string, unknown>;