@arizeai/phoenix-evals 0.5.0 → 0.5.1

package/src/template/createTemplateVariablesProxy.ts

@@ -3,17 +3,17 @@
  * This allows Mustache to access properties of objects (e.g., {{user.name}}) while
  * stringifying objects that are accessed as leaf values (e.g., {{user.profile}}).
  */
-export function createTemplateVariablesProxy(obj: unknown): unknown {
+export function createTemplateVariablesProxy<T>(obj: T): T {
   if (obj === null || obj === undefined) {
     return obj;
   }
 
   if (Array.isArray(obj)) {
-    return obj.map(createTemplateVariablesProxy);
+    return obj.map(createTemplateVariablesProxy) as T;
   }
 
   if (typeof obj === "object") {
-    return new Proxy(obj as Record<string, unknown>, {
+    return new Proxy(obj, {
       get(target, prop: string | symbol) {
         // Handle toString and valueOf to stringify the object when accessed directly
         if (prop === "toString") {

package/src/template/getTemplateVariables.ts

@@ -1,9 +1,9 @@
-import { Template } from "../types/templating";
+import { PromptTemplate } from "../types/templating";
 
 import Mustache from "mustache";
 
 type GetTemplateVariableArgs = {
-  template: Template;
+  template: PromptTemplate;
 };
 /**
  * Parse out the template variables of a prompt
@@ -12,6 +12,22 @@ type GetTemplateVariableArgs = {
  */
 export function getTemplateVariables(args: GetTemplateVariableArgs): string[] {
   const { template } = args;
+  if (typeof template === "string") {
+    return getTemplateVariablesFromString(template);
+  }
+  return template.reduce((acc, message) => {
+    if (message.content !== undefined && typeof message.content === "string") {
+      return [...acc, ...getTemplateVariablesFromString(message.content)];
+    }
+    return acc;
+  }, [] as string[]);
+}
+/**
+ * Parse out the template variables of a string template
+ * @param template - The template to get the variables from
+ * @returns {string[]} a list of prompt template variables
+ */
+function getTemplateVariablesFromString(template: string): string[] {
   const templateSpans = Mustache.parse(template);
   return templateSpans.reduce((acc, templateSpan) => {
     const [spanType, value] = templateSpan;

package/src/types/evals.ts

@@ -1,5 +1,5 @@
 import { WithTelemetry } from "./otel";
-import { Template } from "./templating";
+import { PromptTemplate } from "./templating";
 
 import { LanguageModel } from "ai";
 
@@ -78,7 +78,7 @@ export interface CreateClassifierArgs extends WithTelemetry {
   /**
    * The prompt template to use for classification
    */
-  promptTemplate: Template;
+  promptTemplate: PromptTemplate;
 }
 
 export interface CreateEvaluatorArgs {

package/src/types/templating.ts

@@ -1,10 +1,13 @@
-export type Template = string;
+import type { ModelMessage } from "ai";
+
+export type PromptTemplate = string | Array<ModelMessage>;
+export type RenderedPrompt = string | Array<ModelMessage>;
 export type TemplateVariables = Record<string, unknown>;
 
-/**
- * A class or object that has a prompt template
- */
 export interface WithPromptTemplate {
-  readonly promptTemplate: Template;
-  get promptTemplateVariables(): string[];
+  readonly promptTemplate: PromptTemplate;
+  /**
+   * List out the prompt template variables needed to perform evaluation
+   */
+  readonly promptTemplateVariables: string[];
 }