@arizeai/phoenix-evals 0.5.1 → 0.6.1

package/dist/src/default_templates/HALLUCINATION_TEMPLATE.js

@@ -1,34 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HALLUCINATION_CHOICES = exports.HALLUCINATION_TEMPLATE = void 0;
-exports.HALLUCINATION_TEMPLATE = `
-In this task, you will be presented with a query, a reference text and an answer. The answer is
-generated to the question based on the reference text. The answer may contain false information. You
-must use the reference text to determine if the answer to the question contains false information,
-if the answer is a hallucination of facts. Your objective is to determine whether the answer text
-contains factual information and is not a hallucination. A 'hallucination' refers to
-an answer that is not based on the reference text or assumes information that is not available in
-the reference text. Your response should be a single word: either "factual" or "hallucinated", and
-it should not include any other text or characters. "hallucinated" indicates that the answer
-provides factually inaccurate information to the query based on the reference text. "factual"
-indicates that the answer to the question is correct relative to the reference text, and does not
-contain made up information. Please read the query and reference text carefully before determining
-your response.
-
-    [BEGIN DATA]
-    ************
-    [Query]: {{input}}
-    ************
-    [Reference text]: {{reference}}
-    ************
-    [Answer]: {{output}}
-    ************
-    [END DATA]
-
-Is the answer above factual or hallucinated based on the query and reference text?
-`;
-exports.HALLUCINATION_CHOICES = {
-    hallucinated: 1,
-    factual: 0,
-};
-//# sourceMappingURL=HALLUCINATION_TEMPLATE.js.map

package/dist/src/default_templates/HALLUCINATION_TEMPLATE.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"HALLUCINATION_TEMPLATE.js","sourceRoot":"","sources":["../../../src/default_templates/HALLUCINATION_TEMPLATE.ts"],"names":[],"mappings":";;;AAAa,QAAA,sBAAsB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;CAyBrC,CAAC;AAEW,QAAA,qBAAqB,GAAG;IACnC,YAAY,EAAE,CAAC;IACf,OAAO,EAAE,CAAC;CACX,CAAC"}

package/dist/src/default_templates/index.d.ts

@@ -1,3 +0,0 @@
-export * from "./HALLUCINATION_TEMPLATE";
-export * from "./DOCUMENT_RELEVANCY_TEMPLATE";
-//# sourceMappingURL=index.d.ts.map

package/dist/src/default_templates/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/default_templates/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,+BAA+B,CAAC"}

package/dist/src/default_templates/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/default_templates/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAyC;AACzC,gEAA8C"}

package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts

@@ -1,43 +0,0 @@
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
-export interface DocumentRelevancyEvaluatorArgs extends Omit<CreateClassificationEvaluatorArgs, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
-    optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
-    name?: CreateClassificationEvaluatorArgs["name"];
-    choices?: CreateClassificationEvaluatorArgs["choices"];
-    promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
-}
-/**
- * A record to be evaluated by the document relevancy evaluator.
- */
-export interface DocumentRelevancyEvaluationRecord {
-    input: string;
-    documentText: string;
-    [key: string]: unknown;
-}
-/**
- * Creates a document relevancy evaluator function.
- *
- * This function returns an evaluator that determines whether a given document text
- * is relevant to a provided input question. The evaluator uses a classification model
- * and a prompt template to make its determination.
- *
- * @param args - The arguments for creating the document relevancy evaluator.
- * @param args.model - The model to use for classification.
- * @param args.choices - The possible classification choices (defaults to DOCUMENT_RELEVANCY_CHOICES).
- * @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCY_TEMPLATE).
- * @param args.telemetry - The telemetry to use for the evaluator.
- *
- * @returns An evaluator function that takes a {@link DocumentRelevancyExample} and returns a classification result
- * indicating whether the document is relevant to the input question.
- *
- * @example
- * ```ts
- * const evaluator = createDocumentRelevancyEvaluator({ model: openai("gpt-4o-mini") });
- * const result = await evaluator.evaluate({
- *   input: "What is the capital of France?",
- *   documentText: "Paris is the capital and most populous city of France.",
- * });
- * console.log(result.label); // "relevant" or "unrelated"
- * ```
- */
-export declare function createDocumentRelevancyEvaluator<RecordType extends Record<string, unknown> = DocumentRelevancyEvaluationRecord>(args: DocumentRelevancyEvaluatorArgs): Evaluator<RecordType>;
-//# sourceMappingURL=createDocumentRelevancyEvaluator.d.ts.map

package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAI9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,EACrC,IAAI,EAAE,8BAA8B,GAAG,SAAS,CAAC,UAAU,CAAC,CAiB7D"}

package/dist/src/llm/createDocumentRelevancyEvaluator.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAsDA,4EAsBC;AA5ED,kGAG0D;AAG1D,mFAAgF;AAsBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,gCAAgC,CAK9C,IAAoC;IACpC,MAAM,EACJ,OAAO,GAAG,wDAA0B,EACpC,cAAc,GAAG,yDAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,KAEzB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IAET,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}

package/src/default_templates/DOCUMENT_RELEVANCY_TEMPLATE.ts

@@ -1,25 +0,0 @@
-export const DOCUMENT_RELEVANCY_TEMPLATE = `
-You are comparing a document to a question and trying to determine if the document text
-contains information relevant to answering the question. Here is the data:
-
-    [BEGIN DATA]
-    ************
-    [Question]: {{input}}
-    ************
-    [Document text]: {{documentText}}
-    ************
-    [END DATA]
-
-Compare the Question above to the Document text. You must determine whether the Document text
-contains information that can answer the Question. Please focus on whether the very specific
-question can be answered by the information in the Document text.
-Your response must be single word, either "relevant" or "unrelated",
-and should not contain any text or characters aside from that word.
-"unrelated" means that the document text does not contain an answer to the Question.
-"relevant" means the document text contains an answer to the Question.
-`;
-
-export const DOCUMENT_RELEVANCY_CHOICES = {
-  relevant: 1,
-  unrelated: 0,
-};

package/src/default_templates/HALLUCINATION_TEMPLATE.ts

@@ -1,31 +0,0 @@
-export const HALLUCINATION_TEMPLATE = `
-In this task, you will be presented with a query, a reference text and an answer. The answer is
-generated to the question based on the reference text. The answer may contain false information. You
-must use the reference text to determine if the answer to the question contains false information,
-if the answer is a hallucination of facts. Your objective is to determine whether the answer text
-contains factual information and is not a hallucination. A 'hallucination' refers to
-an answer that is not based on the reference text or assumes information that is not available in
-the reference text. Your response should be a single word: either "factual" or "hallucinated", and
-it should not include any other text or characters. "hallucinated" indicates that the answer
-provides factually inaccurate information to the query based on the reference text. "factual"
-indicates that the answer to the question is correct relative to the reference text, and does not
-contain made up information. Please read the query and reference text carefully before determining
-your response.
-
-    [BEGIN DATA]
-    ************
-    [Query]: {{input}}
-    ************
-    [Reference text]: {{reference}}
-    ************
-    [Answer]: {{output}}
-    ************
-    [END DATA]
-
-Is the answer above factual or hallucinated based on the query and reference text?
-`;
-
-export const HALLUCINATION_CHOICES = {
-  hallucinated: 1,
-  factual: 0,
-};

package/src/default_templates/index.ts

@@ -1,2 +0,0 @@
-export * from "./HALLUCINATION_TEMPLATE";
-export * from "./DOCUMENT_RELEVANCY_TEMPLATE";

package/src/llm/createDocumentRelevancyEvaluator.ts

@@ -1,77 +0,0 @@
-import {
-  DOCUMENT_RELEVANCY_CHOICES,
-  DOCUMENT_RELEVANCY_TEMPLATE,
-} from "../default_templates/DOCUMENT_RELEVANCY_TEMPLATE";
-import { CreateClassificationEvaluatorArgs, Evaluator } from "../types/evals";
-
-import { createClassificationEvaluator } from "./createClassificationEvaluator";
-
-export interface DocumentRelevancyEvaluatorArgs
-  extends Omit<
-    CreateClassificationEvaluatorArgs,
-    "promptTemplate" | "choices" | "optimizationDirection" | "name"
-  > {
-  optimizationDirection?: CreateClassificationEvaluatorArgs["optimizationDirection"];
-  name?: CreateClassificationEvaluatorArgs["name"];
-  choices?: CreateClassificationEvaluatorArgs["choices"];
-  promptTemplate?: CreateClassificationEvaluatorArgs["promptTemplate"];
-}
-
-/**
- * A record to be evaluated by the document relevancy evaluator.
- */
-export interface DocumentRelevancyEvaluationRecord {
-  input: string;
-  documentText: string;
-  [key: string]: unknown;
-}
-
-/**
- * Creates a document relevancy evaluator function.
- *
- * This function returns an evaluator that determines whether a given document text
- * is relevant to a provided input question. The evaluator uses a classification model
- * and a prompt template to make its determination.
- *
- * @param args - The arguments for creating the document relevancy evaluator.
- * @param args.model - The model to use for classification.
- * @param args.choices - The possible classification choices (defaults to DOCUMENT_RELEVANCY_CHOICES).
- * @param args.promptTemplate - The prompt template to use (defaults to DOCUMENT_RELEVANCY_TEMPLATE).
- * @param args.telemetry - The telemetry to use for the evaluator.
- *
- * @returns An evaluator function that takes a {@link DocumentRelevancyExample} and returns a classification result
- * indicating whether the document is relevant to the input question.
- *
- * @example
- * ```ts
- * const evaluator = createDocumentRelevancyEvaluator({ model: openai("gpt-4o-mini") });
- * const result = await evaluator.evaluate({
- *   input: "What is the capital of France?",
- *   documentText: "Paris is the capital and most populous city of France.",
- * });
- * console.log(result.label); // "relevant" or "unrelated"
- * ```
- */
-export function createDocumentRelevancyEvaluator<
-  RecordType extends Record<
-    string,
-    unknown
-  > = DocumentRelevancyEvaluationRecord,
->(args: DocumentRelevancyEvaluatorArgs): Evaluator<RecordType> {
-  const {
-    choices = DOCUMENT_RELEVANCY_CHOICES,
-    promptTemplate = DOCUMENT_RELEVANCY_TEMPLATE,
-    optimizationDirection = "MAXIMIZE",
-    name = "document_relevancy",
-    ...rest
-  } = args;
-
-  return createClassificationEvaluator<RecordType>({
-    ...args,
-    promptTemplate,
-    choices,
-    optimizationDirection,
-    name,
-    ...rest,
-  });
-}