@arizeai/phoenix-evals 0.4.0 → 0.5.1

package/src/template/createTemplateVariablesProxy.ts

@@ -0,0 +1,69 @@
+/**
+ * Creates a Proxy that wraps an object to stringify nested object values when accessed directly.
+ * This allows Mustache to access properties of objects (e.g., {{user.name}}) while
+ * stringifying objects that are accessed as leaf values (e.g., {{user.profile}}).
+ */
+export function createTemplateVariablesProxy<T>(obj: T): T {
+  if (obj === null || obj === undefined) {
+    return obj;
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map(createTemplateVariablesProxy) as T;
+  }
+
+  if (typeof obj === "object") {
+    return new Proxy(obj, {
+      get(target, prop: string | symbol) {
+        // Handle toString and valueOf to stringify the object when accessed directly
+        if (prop === "toString") {
+          return () => JSON.stringify(target);
+        }
+        if (prop === "valueOf") {
+          return () => JSON.stringify(target);
+        }
+
+        if (typeof prop !== "string") {
+          return Reflect.get(target, prop);
+        }
+
+        const value = Reflect.get(target, prop);
+
+        // If the value is an object (not array, not null), wrap it in a proxy
+        // so it can be stringified if accessed directly, or have its properties accessed
+        if (
+          value !== null &&
+          typeof value === "object" &&
+          !Array.isArray(value)
+        ) {
+          return createTemplateVariablesProxy(value);
+        }
+
+        return value;
+      },
+      // Override valueOf and toString to stringify the object when Mustache tries to render it directly
+      // Mustache will call toString() when it needs to render an object as a string
+      has(target, prop) {
+        if (prop === "toString" || prop === "valueOf") {
+          return true;
+        }
+        return Reflect.has(target, prop);
+      },
+      ownKeys(target) {
+        return Reflect.ownKeys(target);
+      },
+      getOwnPropertyDescriptor(target, prop) {
+        if (prop === "toString") {
+          return {
+            enumerable: false,
+            configurable: true,
+            value: () => JSON.stringify(target),
+          };
+        }
+        return Reflect.getOwnPropertyDescriptor(target, prop);
+      },
+    });
+  }
+
+  return obj;
+}

package/src/template/getTemplateVariables.ts

@@ -1,9 +1,9 @@
-import { Template } from "../types/templating";
+import { PromptTemplate } from "../types/templating";
 
 import Mustache from "mustache";
 
 type GetTemplateVariableArgs = {
-  template: Template;
+  template: PromptTemplate;
 };
 /**
  * Parse out the template variables of a prompt
@@ -12,6 +12,22 @@ type GetTemplateVariableArgs = {
  */
 export function getTemplateVariables(args: GetTemplateVariableArgs): string[] {
   const { template } = args;
+  if (typeof template === "string") {
+    return getTemplateVariablesFromString(template);
+  }
+  return template.reduce((acc, message) => {
+    if (message.content !== undefined && typeof message.content === "string") {
+      return [...acc, ...getTemplateVariablesFromString(message.content)];
+    }
+    return acc;
+  }, [] as string[]);
+}
+/**
+ * Parse out the template variables of a string template
+ * @param template - The template to get the variables from
+ * @returns {string[]} a list of prompt template variables
+ */
+function getTemplateVariablesFromString(template: string): string[] {
   const templateSpans = Mustache.parse(template);
   return templateSpans.reduce((acc, templateSpan) => {
     const [spanType, value] = templateSpan;

package/src/types/evals.ts

@@ -1,4 +1,5 @@
 import { WithTelemetry } from "./otel";
+import { PromptTemplate } from "./templating";
 
 import { LanguageModel } from "ai";
 
@@ -77,7 +78,7 @@ export interface CreateClassifierArgs extends WithTelemetry {
   /**
    * The prompt template to use for classification
    */
-  promptTemplate: string;
+  promptTemplate: PromptTemplate;
 }
 
 export interface CreateEvaluatorArgs {

package/src/types/templating.ts

@@ -1,2 +1,13 @@
-export type Template = string;
+import type { ModelMessage } from "ai";
+
+export type PromptTemplate = string | Array<ModelMessage>;
+export type RenderedPrompt = string | Array<ModelMessage>;
 export type TemplateVariables = Record<string, unknown>;
+
+export interface WithPromptTemplate {
+  readonly promptTemplate: PromptTemplate;
+  /**
+   * List out the prompt template variables needed to perform evaluation
+   */
+  readonly promptTemplateVariables: string[];
+}