@arizeai/phoenix-evals 1.0.0 → 1.0.2

package/README.md

@@ -91,25 +91,58 @@ See the complete example in [`examples/classifier_example.ts`](examples/classifi
 
 The library includes several pre-built evaluators for common evaluation tasks. These evaluators come with optimized prompts and can be used directly with any AI SDK model.
 
+All pre-built evaluators are available from the `@arizeai/phoenix-evals/llm` module:
+
+| Evaluator              | Function                              | Description                                                                       |
+| ---------------------- | ------------------------------------- | --------------------------------------------------------------------------------- |
+| Faithfulness           | `createFaithfulnessEvaluator`         | Detects hallucinations — checks if the output is grounded in the provided context |
+| Conciseness            | `createConcisenessEvaluator`          | Evaluates whether the response is appropriately concise                           |
+| Correctness            | `createCorrectnessEvaluator`          | Checks if the output is factually correct given the input                         |
+| Document Relevance     | `createDocumentRelevanceEvaluator`    | Measures how relevant a retrieved document is to the query                        |
+| Refusal                | `createRefusalEvaluator`              | Detects whether the model refused to answer                                       |
+| Tool Invocation        | `createToolInvocationEvaluator`       | Evaluates whether the correct tool was invoked with the right arguments           |
+| Tool Selection         | `createToolSelectionEvaluator`        | Checks whether the right tool was selected for the task                           |
+| Tool Response Handling | `createToolResponseHandlingEvaluator` | Evaluates how well the model uses a tool's response                               |
+
 ```typescript
-import { createFaithfulnessEvaluator } from "@arizeai/phoenix-evals/llm";
+import {
+  createFaithfulnessEvaluator,
+  createConcisenessEvaluator,
+  createCorrectnessEvaluator,
+  createDocumentRelevanceEvaluator,
+  createRefusalEvaluator,
+} from "@arizeai/phoenix-evals/llm";
 import { openai } from "@ai-sdk/openai";
-const model = openai("gpt-4o-mini");
 
-// Faithfulness Detection
-const faithfulnessEvaluator = createFaithfulnessEvaluator({
-  model,
-});
+const model = openai("gpt-4o-mini");
 
-// Use the evaluators
-const result = await faithfulnessEvaluator({
+// Faithfulness: checks if the output is grounded in the context
+const faithfulnessEvaluator = createFaithfulnessEvaluator({ model });
+const faithfulnessResult = await faithfulnessEvaluator.evaluate({
   input: "What is the capital of France?",
   context: "France is a country in Europe. Paris is its capital city.",
   output: "The capital of France is London.",
 });
-
-console.log(result);
+console.log(faithfulnessResult);
 // Output: { label: "unfaithful", score: 0, explanation: "..." }
+
+// Correctness: checks if the output is factually correct
+const correctnessEvaluator = createCorrectnessEvaluator({ model });
+const correctnessResult = await correctnessEvaluator.evaluate({
+  input: "What is the capital of France?",
+  output: "Paris is the capital of France.",
+});
+console.log(correctnessResult);
+// Output: { label: "correct", score: 1, explanation: "..." }
+
+// Document Relevance: checks if a retrieved document is relevant to the query
+const relevanceEvaluator = createDocumentRelevanceEvaluator({ model });
+const relevanceResult = await relevanceEvaluator.evaluate({
+  input: "What is the capital of France?",
+  documentText: "Paris is the capital of France and a major European city.",
+});
+console.log(relevanceResult);
+// Output: { label: "relevant", score: 1, explanation: "..." }
 ```
 
 ### Data Mapping
@@ -117,10 +150,8 @@ console.log(result);
 When your data structure doesn't match what an evaluator expects, use `bindEvaluator` to map your fields to the evaluator's expected input format:
 
 ```typescript
-import {
-  bindEvaluator,
-  createFaithfulnessEvaluator,
-} from "@arizeai/phoenix-evals";
+import { bindEvaluator } from "@arizeai/phoenix-evals";
+import { createFaithfulnessEvaluator } from "@arizeai/phoenix-evals/llm";
 import { openai } from "@ai-sdk/openai";
 
 const model = openai("gpt-4o-mini");

package/dist/esm/helpers/createEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createEvaluator.d.ts","sourceRoot":"","sources":["../../../src/helpers/createEvaluator.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EACV,cAAc,EACd,qBAAqB,EACrB,eAAe,EAChB,MAAM,UAAU,CAAC;AAIlB,KAAK,KAAK,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;AAMrC;;;;GAIG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC;;;;;;;;;;OAUG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;;;;;;OAYG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB;;;;;;;;;;;;;;OAcG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,wBAAgB,eAAe,CAC7B,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACpE,EAAE,SAAS,KAAK,GAAG,KAAK,EACxB,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,aAAa,CAAC,UAAU,CAAC,CAyBrE"}
+{"version":3,"file":"createEvaluator.d.ts","sourceRoot":"","sources":["../../../src/helpers/createEvaluator.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG3D,OAAO,KAAK,EACV,cAAc,EACd,qBAAqB,EACrB,eAAe,EAChB,MAAM,UAAU,CAAC;AAIlB,KAAK,KAAK,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;AAMrC;;;;GAIG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC;;;;;;;;;;OAUG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;;;;;;OAYG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB;;;;;;;;;;;;;;OAcG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,wBAAgB,eAAe,CAC7B,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACpE,EAAE,SAAS,KAAK,GAAG,KAAK,EACxB,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,aAAa,CAAC,UAAU,CAAC,CAyBrE"}

package/dist/esm/helpers/createEvaluator.js

@@ -1,5 +1,6 @@
 import { withSpan } from "@arizeai/openinference-core";
 import { FunctionEvaluator } from "../core/FunctionEvaluator.js";
+import { tracer as defaultTracer } from "../telemetry/index.js";
 import { asEvaluatorFn } from "./asEvaluatorFn.js";
 function generateUniqueName() {
     return `evaluator-${Math.random().toString(36).substring(2, 15)}`;
@@ -80,7 +81,7 @@ export function createEvaluator(fn, options) {
     // Add OpenTelemetry span wrapping if telemetry is enabled
     if (telemetry && telemetry.isEnabled) {
         evaluateFn = withSpan(evaluateFn, {
-            tracer: telemetry.tracer,
+            tracer: telemetry.tracer ?? defaultTracer,
             name: evaluatorName,
             kind: "EVALUATOR",
         });

package/dist/esm/helpers/createEvaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"createEvaluator.js","sourceRoot":"","sources":["../../../src/helpers/createEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAGvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAM9D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAKhD,SAAS,kBAAkB;IACzB,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACpE,CAAC;AAqED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,MAAM,UAAU,eAAe,CAG7B,EAAM,EAAE,OAAgC;IACxC,MAAM,EACJ,IAAI,EACJ,IAAI,EACJ,qBAAqB,EACrB,SAAS,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,GAChC,GAAG,OAAO,IAAI,EAAE,CAAC;IAClB,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,kBAAkB,EAAE,CAAC;IAC9D,IAAI,UAAU,GAAG,aAAa,CAAa,EAAE,CAAC,CAAC;IAE/C,0DAA0D;IAC1D,IAAI,SAAS,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;QACrC,UAAU,GAAG,QAAQ,CAAC,UAAU,EAAE;YAChC,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,IAAI,EAAE,aAAa;YACnB,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,iBAAiB,CAAa;QACvC,UAAU;QACV,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,IAAI,IAAI,MAAM;QACpB,qBAAqB,EAAE,qBAAqB,IAAI,UAAU;QAC1D,SAAS;KACV,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"createEvaluator.js","sourceRoot":"","sources":["../../../src/helpers/createEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAGvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,cAAc,CAAC;AAMvD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAKhD,SAAS,kBAAkB;IACzB,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACpE,CAAC;AAqED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,MAAM,UAAU,eAAe,CAG7B,EAAM,EAAE,OAAgC;IACxC,MAAM,EACJ,IAAI,EACJ,IAAI,EACJ,qBAAqB,EACrB,SAAS,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,GAChC,GAAG,OAAO,IAAI,EAAE,CAAC;IAClB,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,kBAAkB,EAAE,CAAC;IAC9D,IAAI,UAAU,GAAG,aAAa,CAAa,EAAE,CAAC,CAAC;IAE/C,0DAA0D;IAC1D,IAAI,SAAS,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;QACrC,UAAU,GAAG,QAAQ,CAAC,UAAU,EAAE;YAChC,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,aAAa;YACzC,IAAI,EAAE,aAAa;YACnB,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,iBAAiB,CAAa;QACvC,UAAU;QACV,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,IAAI,IAAI,MAAM;QACpB,qBAAqB,EAAE,qBAAqB,IAAI,UAAU;QAC1D,SAAS;KACV,CAAC,CAAC;AACL,CAAC"}

package/dist/esm/llm/ClassificationEvaluator.d.ts

@@ -21,7 +21,7 @@ export declare class ClassificationEvaluator<RecordType extends Record<string, u
      */
     readonly choices: ClassificationChoicesMap;
     constructor(args: CreateClassificationEvaluatorArgs<RecordType>);
-    evaluate: (example: RecordType) => Promise<import("../types/index.js").EvaluationResult>;
+    evaluate: (example: RecordType) => Promise<import("../index.js").EvaluationResult>;
     /**
      * List out the prompt template variables needed to perform evaluation
      */

package/dist/esm/llm/ClassificationEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EACV,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;gBAE/B,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC;IAS/D,QAAQ,GAAI,SAAS,UAAU,kDAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC;CAMvC"}
+{"version":3,"file":"ClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/ClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,OAAO,KAAK,EACV,wBAAwB,EACxB,iCAAiC,EACjC,WAAW,EACX,cAAc,EACd,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;GAEG;AACH,qBAAa,uBAAuB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7E,SAAQ,YAAY,CAAC,UAAU,CAC/B,YAAW,kBAAkB;IAE7B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;OAEG;IACH,OAAO,CAAC,wBAAwB,CAAuB;IACvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,wBAAwB,CAAC;gBAE/B,IAAI,EAAE,iCAAiC,CAAC,UAAU,CAAC;IAS/D,QAAQ,GAAI,SAAS,UAAU,4CAM7B;IACF;;OAEG;IACH,IAAI,uBAAuB,IAAI,MAAM,EAAE,CAStC;IACD;;OAEG;IACH,gBAAgB,CACd,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,GACtC,uBAAuB,CAAC,UAAU,CAAC;CAMvC"}

package/dist/esm/telemetry/index.d.ts

@@ -1,2 +1,11 @@
-export declare const tracer: import("@opentelemetry/api").Tracer;
+import { type Tracer } from "@opentelemetry/api";
+/**
+ * Returns a lazy tracer that resolves from `trace.getTracer()` on every call,
+ * so evaluator spans follow whichever provider is currently mounted as global.
+ *
+ * Cast to `Tracer` is necessary because `startActiveSpan` has multiple
+ * overload signatures that cannot be satisfied by a single implementation.
+ */
+export declare function getTracer(name?: string): Tracer;
+export declare const tracer: Tracer;
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/telemetry/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,MAAM,qCAAmC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAIxD;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,IAAI,GAAE,MAA4B,GAAG,MAAM,CAUpE;AAED,eAAO,MAAM,MAAM,QAAc,CAAC"}

package/dist/esm/telemetry/index.js

@@ -1,3 +1,22 @@
 import { trace } from "@opentelemetry/api";
-export const tracer = trace.getTracer("phoenix-evals");
+const DEFAULT_TRACER_NAME = "phoenix-evals";
+/**
+ * Returns a lazy tracer that resolves from `trace.getTracer()` on every call,
+ * so evaluator spans follow whichever provider is currently mounted as global.
+ *
+ * Cast to `Tracer` is necessary because `startActiveSpan` has multiple
+ * overload signatures that cannot be satisfied by a single implementation.
+ */
+export function getTracer(name = DEFAULT_TRACER_NAME) {
+    return {
+        startSpan(spanName, options, context) {
+            return trace.getTracer(name).startSpan(spanName, options, context);
+        },
+        startActiveSpan(...args) {
+            const tracer = trace.getTracer(name);
+            return Reflect.apply(tracer.startActiveSpan, tracer, args);
+        },
+    };
+}
+export const tracer = getTracer();
 //# sourceMappingURL=index.js.map

package/dist/esm/telemetry/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAe,MAAM,oBAAoB,CAAC;AAExD,MAAM,mBAAmB,GAAG,eAAe,CAAC;AAE5C;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,OAAe,mBAAmB;IAC1D,OAAO;QACL,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO;YAClC,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACrE,CAAC;QACD,eAAe,CAAC,GAAG,IAAe;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrC,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,CAAC;KACQ,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC"}