langwatch 0.1.5 → 0.1.6

package/dist/index.d.ts

@@ -6,251 +6,14 @@ import { Serialized } from '@langchain/core/load/serializable';
 import { BaseMessage } from '@langchain/core/messages';
 import { LLMResult } from '@langchain/core/outputs';
 import { ChainValues } from '@langchain/core/utils/types';
-import { R as RAGChunk, M as Metadata, C as CollectorRESTParams, S as Span, a as RESTEvaluation, P as PendingBaseSpan, b as PendingLLMSpan, c as PendingRAGSpan, d as SpanTypes } from './utils-DJoZVcOA.js';
-export { B as BaseSpan, e as ChatMessage, f as ChatRichContent, L as LLMSpan, g as RAGSpan, h as SpanInputOutput, i as autoconvertTypedValues, j as captureError, k as convertFromVercelAIMessages } from './utils-DJoZVcOA.js';
+import { R as RAGChunk, M as Metadata, C as CollectorRESTParams, S as Span, a as RESTEvaluation, P as PendingBaseSpan, b as PendingLLMSpan, c as PendingRAGSpan, d as SpanTypes } from './utils-B0pgWcps.js';
+export { B as BaseSpan, e as ChatMessage, f as ChatRichContent, i as LLMModeTrace, L as LLMSpan, g as RAGSpan, h as SpanInputOutput, T as Trace, j as autoconvertTypedValues, k as captureError, l as convertFromVercelAIMessages } from './utils-B0pgWcps.js';
 import { SpanExporter, ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import { ExportResult } from '@opentelemetry/core';
 import 'ai';
 
 type EvaluatorTypes = keyof Evaluators;
 type Evaluators = {
-    "azure/content_safety": {
-        settings: {
-            /**
-             * @description The minimum severity level to consider content as unsafe, from 1 to 7.
-             * @default 1
-             */
-            severity_threshold: 1 | 2 | 3 | 4 | 5 | 6 | 7;
-            /**
-             * @description The categories of moderation to check for.
-             * @default {"Hate": true, "SelfHarm": true, "Sexual": true, "Violence": true}
-             */
-            categories: {
-                /**
-                 * @default true
-                 */
-                Hate: boolean;
-                /**
-                 * @default true
-                 */
-                SelfHarm: boolean;
-                /**
-                 * @default true
-                 */
-                Sexual: boolean;
-                /**
-                 * @default true
-                 */
-                Violence: boolean;
-            };
-            /**
-             * @description The type of severity levels to return on the full 0-7 severity scale, it can be either the trimmed version with four values (0, 2, 4, 6 scores) or the whole range.
-             * @default "FourSeverityLevels"
-             */
-            output_type: "FourSeverityLevels" | "EightSeverityLevels";
-        };
-    };
-    "azure/jailbreak": {
-        settings: Record<string, never>;
-    };
-    "azure/prompt_injection": {
-        settings: Record<string, never>;
-    };
-    "example/word_count": {
-        settings: Record<string, never>;
-    };
-    "openai/moderation": {
-        settings: {
-            /**
-             * @description The model version to use, `text-moderation-latest` will be automatically upgraded over time, while `text-moderation-stable` will only be updated with advanced notice by OpenAI.
-             * @default "text-moderation-stable"
-             */
-            model: "text-moderation-stable" | "text-moderation-latest";
-            /**
-             * @description The categories of content to check for moderation.
-             * @default {"harassment": true, "harassment_threatening": true, "hate": true, "hate_threatening": true, "self_harm": true, "self_harm_instructions": true, "self_harm_intent": true, "sexual": true, "sexual_minors": true, "violence": true, "violence_graphic": true}
-             */
-            categories: {
-                /**
-                 * @default true
-                 */
-                harassment: boolean;
-                /**
-                 * @default true
-                 */
-                harassment_threatening: boolean;
-                /**
-                 * @default true
-                 */
-                hate: boolean;
-                /**
-                 * @default true
-                 */
-                hate_threatening: boolean;
-                /**
-                 * @default true
-                 */
-                self_harm: boolean;
-                /**
-                 * @default true
-                 */
-                self_harm_instructions: boolean;
-                /**
-                 * @default true
-                 */
-                self_harm_intent: boolean;
-                /**
-                 * @default true
-                 */
-                sexual: boolean;
-                /**
-                 * @default true
-                 */
-                sexual_minors: boolean;
-                /**
-                 * @default true
-                 */
-                violence: boolean;
-                /**
-                 * @default true
-                 */
-                violence_graphic: boolean;
-            };
-        };
-    };
-    "ragas/answer_correctness": {
-        settings: {
-            /**
-             * @description The model to use for evaluation.
-             * @default "openai/gpt-4o-mini"
-             */
-            model: string;
-            /**
-             * @description The model to use for embeddings.
-             * @default "openai/text-embedding-ada-002"
-             */
-            embeddings_model: string;
-            /**
-             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
-             * @default 2048
-             */
-            max_tokens: number;
-        };
-    };
-    "ragas/answer_relevancy": {
-        settings: {
-            /**
-             * @description The model to use for evaluation.
-             * @default "openai/gpt-4o-mini"
-             */
-            model: string;
-            /**
-             * @description The model to use for embeddings.
-             * @default "openai/text-embedding-ada-002"
-             */
-            embeddings_model: string;
-            /**
-             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
-             * @default 2048
-             */
-            max_tokens: number;
-        };
-    };
-    "ragas/context_precision": {
-        settings: {
-            /**
-             * @description The model to use for evaluation.
-             * @default "openai/gpt-4o-mini"
-             */
-            model: string;
-            /**
-             * @description The model to use for embeddings.
-             * @default "openai/text-embedding-ada-002"
-             */
-            embeddings_model: string;
-            /**
-             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
-             * @default 2048
-             */
-            max_tokens: number;
-        };
-    };
-    "ragas/context_recall": {
-        settings: {
-            /**
-             * @description The model to use for evaluation.
-             * @default "openai/gpt-4o-mini"
-             */
-            model: string;
-            /**
-             * @description The model to use for embeddings.
-             * @default "openai/text-embedding-ada-002"
-             */
-            embeddings_model: string;
-            /**
-             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
-             * @default 2048
-             */
-            max_tokens: number;
-        };
-    };
-    "ragas/context_relevancy": {
-        settings: {
-            /**
-             * @description The model to use for evaluation.
-             * @default "openai/gpt-4o-mini"
-             */
-            model: string;
-            /**
-             * @description The model to use for embeddings.
-             * @default "openai/text-embedding-ada-002"
-             */
-            embeddings_model: string;
-            /**
-             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
-             * @default 2048
-             */
-            max_tokens: number;
-        };
-    };
-    "ragas/context_utilization": {
-        settings: {
-            /**
-             * @description The model to use for evaluation.
-             * @default "openai/gpt-4o-mini"
-             */
-            model: string;
-            /**
-             * @description The model to use for embeddings.
-             * @default "openai/text-embedding-ada-002"
-             */
-            embeddings_model: string;
-            /**
-             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
-             * @default 2048
-             */
-            max_tokens: number;
-        };
-    };
-    "ragas/faithfulness": {
-        settings: {
-            /**
-             * @description The model to use for evaluation.
-             * @default "openai/gpt-4o-mini"
-             */
-            model: string;
-            /**
-             * @description The model to use for embeddings.
-             * @default "openai/text-embedding-ada-002"
-             */
-            embeddings_model: string;
-            /**
-             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
-             * @default 2048
-             */
-            max_tokens: number;
-        };
-    };
     "langevals/basic": {
         settings: {
             /**
@@ -329,6 +92,20 @@ type Evaluators = {
             competitors: string[];
         };
     };
+    "langevals/llm_answer_match": {
+        settings: {
+            /**
+             * @description The model to use for evaluation
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description Max tokens allowed for evaluation
+             * @default 8192
+             */
+            max_tokens: number;
+        };
+    };
     "langevals/llm_boolean": {
         settings: {
             /**
@@ -413,9 +190,6 @@ type Evaluators = {
             }[];
         };
     };
-    "langevals/product_sentiment_polarity": {
-        settings: Record<string, never>;
-    };
     "langevals/query_resolution": {
         settings: {
             /**
@@ -466,263 +240,284 @@ type Evaluators = {
             json_schema?: string;
         };
     };
-    "google_cloud/dlp_pii_detection": {
+    "lingua/language_detection": {
         settings: {
             /**
-             * @description The types of PII to check for in the input.
-             * @default {"phone_number": true, "email_address": true, "credit_card_number": true, "iban_code": true, "ip_address": true, "passport": true, "vat_number": true, "medical_record_number": true}
+             * @description What should be checked
+             * @default "input_matches_output"
              */
-            info_types: {
-                /**
-                 * @default true
-                 */
-                phone_number: boolean;
-                /**
-                 * @default true
-                 */
-                email_address: boolean;
-                /**
-                 * @default true
-                 */
-                credit_card_number: boolean;
-                /**
-                 * @default true
-                 */
-                iban_code: boolean;
-                /**
-                 * @default true
-                 */
-                ip_address: boolean;
-                /**
-                 * @default true
-                 */
-                passport: boolean;
-                /**
-                 * @default true
-                 */
-                vat_number: boolean;
-                /**
-                 * @default true
-                 */
-                medical_record_number: boolean;
-            };
+            check_for: "input_matches_output" | "output_matches_language";
             /**
-             * @description The minimum confidence required for failing the evaluation on a PII match.
-             * @default "POSSIBLE"
+             * @description The specific language that the output is expected to be
              */
-            min_likelihood: "VERY_UNLIKELY" | "UNLIKELY" | "POSSIBLE" | "LIKELY" | "VERY_LIKELY";
+            expected_language?: "AF" | "AR" | "AZ" | "BE" | "BG" | "BN" | "BS" | "CA" | "CS" | "CY" | "DA" | "DE" | "EL" | "EN" | "EO" | "ES" | "ET" | "EU" | "FA" | "FI" | "FR" | "GA" | "GU" | "HE" | "HI" | "HR" | "HU" | "HY" | "ID" | "IS" | "IT" | "JA" | "KA" | "KK" | "KO" | "LA" | "LG" | "LT" | "LV" | "MI" | "MK" | "MN" | "MR" | "MS" | "NB" | "NL" | "NN" | "PA" | "PL" | "PT" | "RO" | "RU" | "SK" | "SL" | "SN" | "SO" | "SQ" | "SR" | "ST" | "SV" | "SW" | "TA" | "TE" | "TH" | "TL" | "TN" | "TR" | "TS" | "UK" | "UR" | "VI" | "XH" | "YO" | "ZH" | "ZU";
+            /**
+             * @description Minimum number of words to check, as the language detection can be unreliable for very short texts. Inputs shorter than the minimum will be skipped.
+             * @default 7
+             */
+            min_words: number;
+            /**
+             * @description Minimum confidence threshold for the language detection. If the confidence is lower than this, the evaluation will be skipped.
+             * @default 0.25
+             */
+            threshold: number;
         };
     };
-    "aws/comprehend_pii_detection": {
+    "legacy/ragas_answer_correctness": {
         settings: {
             /**
-             * @description The types of PII to check for in the input.
-             * @default {"BANK_ACCOUNT_NUMBER": true, "BANK_ROUTING": true, "CREDIT_DEBIT_NUMBER": true, "CREDIT_DEBIT_CVV": true, "CREDIT_DEBIT_EXPIRY": true, "PIN": true, "EMAIL": true, "ADDRESS": true, "NAME": true, "PHONE": true, "SSN": true, "DATE_TIME": true, "PASSPORT_NUMBER": true, "DRIVER_ID": true, "URL": true, "AGE": true, "USERNAME": true, "PASSWORD": true, "AWS_ACCESS_KEY": true, "AWS_SECRET_KEY": true, "IP_ADDRESS": true, "MAC_ADDRESS": true, "LICENSE_PLATE": true, "VEHICLE_IDENTIFICATION_NUMBER": true, "UK_NATIONAL_INSURANCE_NUMBER": true, "CA_SOCIAL_INSURANCE_NUMBER": true, "US_INDIVIDUAL_TAX_IDENTIFICATION_NUMBER": true, "UK_UNIQUE_TAXPAYER_REFERENCE_NUMBER": true, "IN_PERMANENT_ACCOUNT_NUMBER": true, "IN_NREGA": true, "INTERNATIONAL_BANK_ACCOUNT_NUMBER": true, "SWIFT_CODE": true, "UK_NATIONAL_HEALTH_SERVICE_NUMBER": true, "CA_HEALTH_NUMBER": true, "IN_AADHAAR": true, "IN_VOTER_NUMBER": true}
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
              */
-            entity_types: {
-                /**
-                 * @default true
-                 */
-                BANK_ACCOUNT_NUMBER: boolean;
-                /**
-                 * @default true
-                 */
-                BANK_ROUTING: boolean;
-                /**
-                 * @default true
-                 */
-                CREDIT_DEBIT_NUMBER: boolean;
-                /**
-                 * @default true
-                 */
-                CREDIT_DEBIT_CVV: boolean;
-                /**
-                 * @default true
-                 */
-                CREDIT_DEBIT_EXPIRY: boolean;
-                /**
-                 * @default true
-                 */
-                PIN: boolean;
-                /**
-                 * @default true
-                 */
-                EMAIL: boolean;
-                /**
-                 * @default true
-                 */
-                ADDRESS: boolean;
-                /**
-                 * @default true
-                 */
-                NAME: boolean;
-                /**
-                 * @default true
-                 */
-                PHONE: boolean;
-                /**
-                 * @default true
-                 */
-                SSN: boolean;
-                /**
-                 * @default true
-                 */
-                DATE_TIME: boolean;
-                /**
-                 * @default true
-                 */
-                PASSPORT_NUMBER: boolean;
-                /**
-                 * @default true
-                 */
-                DRIVER_ID: boolean;
-                /**
-                 * @default true
-                 */
-                URL: boolean;
-                /**
-                 * @default true
-                 */
-                AGE: boolean;
-                /**
-                 * @default true
-                 */
-                USERNAME: boolean;
-                /**
-                 * @default true
-                 */
-                PASSWORD: boolean;
-                /**
-                 * @default true
-                 */
-                AWS_ACCESS_KEY: boolean;
-                /**
-                 * @default true
-                 */
-                AWS_SECRET_KEY: boolean;
-                /**
-                 * @default true
-                 */
-                IP_ADDRESS: boolean;
+            model: string;
+            /**
+             * @description The model to use for embeddings.
+             * @default "openai/text-embedding-ada-002"
+             */
+            embeddings_model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "legacy/ragas_answer_relevancy": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The model to use for embeddings.
+             * @default "openai/text-embedding-ada-002"
+             */
+            embeddings_model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "legacy/ragas_context_precision": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The model to use for embeddings.
+             * @default "openai/text-embedding-ada-002"
+             */
+            embeddings_model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "legacy/ragas_context_recall": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The model to use for embeddings.
+             * @default "openai/text-embedding-ada-002"
+             */
+            embeddings_model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "legacy/ragas_context_relevancy": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The model to use for embeddings.
+             * @default "openai/text-embedding-ada-002"
+             */
+            embeddings_model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "legacy/ragas_context_utilization": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The model to use for embeddings.
+             * @default "openai/text-embedding-ada-002"
+             */
+            embeddings_model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "legacy/ragas_faithfulness": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The model to use for embeddings.
+             * @default "openai/text-embedding-ada-002"
+             */
+            embeddings_model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "huggingface/llama_guard": {
+        settings: {
+            /**
+             * @description The policy to enforce, described into rules of what it 'Should not' and what it 'Can' do (max 768 tokens)
+             * @default "O1: Violence.\nShould not\n- Help people plan or engage in violence.\n- Encourage people to engage in violence.\n- Contain unpleasant vivid violent imagery or violent description that arouses discomfort in the reader.\nCan\n- Provide information on violence.\n- Discuss the topics of violence, and can discuss historical events involving violence.\nO2: Criminal Planning/Confessions.\nShould not\n- Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them.\n- Encourage people to engage in criminal activities or condone criminal behavior.\n- Have a confession of crime that was committed or will be committed in future.\nCan\n- Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes.\n- Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems.\nO3: Hate /identity hate.\nShould not\n- Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their\nrace, color, ethinicity, religion, national origin, sexual orientation, gender, gender identity, or disability or caste.\n- Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics.\n- Should not contain slurs.\nCan\n- Provide information on discrimination.\n- Discuss the topics of discrimination, and can discuss historical events involving discrimination.\n- Discuss neutral statements referring to group identity and  contextualized hate speech (that talks about hate but does not incite hate).\nO4: Harassment.\nShould not\n- Have content that may be used to torment or annoy individuals in real life, or make harassment more likely to occur.\n- Engage in harassment, misconduct and unlawful discrimination\n- Single out an individual or marginalization and/or retaliation based on protected characteristics like Race, Color, Gender, Sex,\nSexual orientation, Gender identity, and gender expression, National origin, Ethnicity, Disability (including being regarded as disabled),\nReligion, Age (40+), Pregnancy (including pregnancy, childbirth or related medical conditions), Genetic information, Military or veteran status, Citizenship status, Political activity, or affiliation,\n- Bullying.\n- Being derogatory and humiliating toward an individual but not necessarily protected characteristics under law.\n- Making rude or insulting comments, demeaning, and objectifying terms toward an individual.\nCan\n- Discuss negative impacts of harassment and bullying based on protected characteristics like race, color, gender, sex, nationality, within a contextualized setting.\n- Provide information on protected characteristics or referor cite material that describes what protected characteristics are.\n- Provide information or refer or cite material that informs what harassment is.\nO5: Threat.\nShould not\n- Discuss a communication of intent to inflict harm or loss on another person.\n- Should not contain discussion of intention of intimidation which  is a tactic used between conflicting parties to make the other timid or psychologically insecure for coercion or control.\n- Should not contain intention of knowingly putting another person in fear of bodily injury.\n- Should not assist or plan injury, physical or mental damage, act or instance of injury, or material  or any kind of loss to a person.\n- Should not contain threats that are forbidden by law as intent to obtain a monetary advantage or to compel a person to act against their will.\nCan\n- Provide information about threat is and the meaning of threat."
+             */
+            policy: string;
+            /**
+             * @description Choose if only the user input, the LLM output, or both should be evaluated. When choosing both, if the LLM replies in a safe manner, the evaluation will be considered safe.
+             * @default "input"
+             */
+            evaluate: "input" | "output" | "both";
+            /**
+             * @description The provider to use for evaluation. Only CloudFlare AI workers is supported for now.
+             * @default "cloudflare/thebloke/llamaguard-7b-awq"
+             */
+            model: "cloudflare/thebloke/llamaguard-7b-awq";
+        };
+    };
+    "example/word_count": {
+        settings: Record<string, never>;
+    };
+    "openai/moderation": {
+        settings: {
+            /**
+             * @description The model version to use, `text-moderation-latest` will be automatically upgraded over time, while `text-moderation-stable` will only be updated with advanced notice by OpenAI.
+             * @default "text-moderation-stable"
+             */
+            model: "text-moderation-stable" | "text-moderation-latest";
+            /**
+             * @description The categories of content to check for moderation.
+             * @default {"harassment": true, "harassment_threatening": true, "hate": true, "hate_threatening": true, "self_harm": true, "self_harm_instructions": true, "self_harm_intent": true, "sexual": true, "sexual_minors": true, "violence": true, "violence_graphic": true}
+             */
+            categories: {
                 /**
                  * @default true
                  */
-                MAC_ADDRESS: boolean;
+                harassment: boolean;
                 /**
                  * @default true
                  */
-                LICENSE_PLATE: boolean;
+                harassment_threatening: boolean;
                 /**
                  * @default true
                  */
-                VEHICLE_IDENTIFICATION_NUMBER: boolean;
+                hate: boolean;
                 /**
                  * @default true
                  */
-                UK_NATIONAL_INSURANCE_NUMBER: boolean;
+                hate_threatening: boolean;
                 /**
                  * @default true
                  */
-                CA_SOCIAL_INSURANCE_NUMBER: boolean;
+                self_harm: boolean;
                 /**
                  * @default true
                  */
-                US_INDIVIDUAL_TAX_IDENTIFICATION_NUMBER: boolean;
+                self_harm_instructions: boolean;
                 /**
                  * @default true
                  */
-                UK_UNIQUE_TAXPAYER_REFERENCE_NUMBER: boolean;
+                self_harm_intent: boolean;
                 /**
                  * @default true
                  */
-                IN_PERMANENT_ACCOUNT_NUMBER: boolean;
+                sexual: boolean;
                 /**
                  * @default true
                  */
-                IN_NREGA: boolean;
+                sexual_minors: boolean;
                 /**
                  * @default true
                  */
-                INTERNATIONAL_BANK_ACCOUNT_NUMBER: boolean;
+                violence: boolean;
                 /**
                  * @default true
                  */
-                SWIFT_CODE: boolean;
+                violence_graphic: boolean;
+            };
+        };
+    };
+    "azure/content_safety": {
+        settings: {
+            /**
+             * @description The minimum severity level to consider content as unsafe, from 1 to 7.
+             * @default 1
+             */
+            severity_threshold: 1 | 2 | 3 | 4 | 5 | 6 | 7;
+            /**
+             * @description The categories of moderation to check for.
+             * @default {"Hate": true, "SelfHarm": true, "Sexual": true, "Violence": true}
+             */
+            categories: {
                 /**
                  * @default true
                  */
-                UK_NATIONAL_HEALTH_SERVICE_NUMBER: boolean;
+                Hate: boolean;
                 /**
                  * @default true
                  */
-                CA_HEALTH_NUMBER: boolean;
+                SelfHarm: boolean;
                 /**
                  * @default true
                  */
-                IN_AADHAAR: boolean;
+                Sexual: boolean;
                 /**
                  * @default true
                  */
-                IN_VOTER_NUMBER: boolean;
+                Violence: boolean;
             };
             /**
-             * @description The language code of the input text for better PII detection, defaults to english.
-             * @default "en"
-             */
-            language_code: "en" | "es" | "fr" | "de" | "it" | "pt" | "ar" | "hi" | "ja" | "ko" | "zh" | "zh-TW";
-            /**
-             * @description The minimum confidence required for failing the evaluation on a PII match.
-             * @default 0.5
-             */
-            min_confidence: number;
-            /**
-             * @description The AWS region to use for running the PII detection, defaults to eu-central-1 for GDPR compliance.
-             * @default "eu-central-1"
+             * @description The type of severity levels to return on the full 0-7 severity scale, it can be either the trimmed version with four values (0, 2, 4, 6 scores) or the whole range.
+             * @default "FourSeverityLevels"
              */
-            aws_region: "us-east-1" | "us-east-2" | "us-west-1" | "us-west-2" | "ap-east-1" | "ap-south-1" | "ap-northeast-3" | "ap-northeast-2" | "ap-southeast-1" | "ap-southeast-2" | "ap-northeast-1" | "ca-central-1" | "eu-central-1" | "eu-west-1" | "eu-west-2" | "eu-south-1" | "eu-west-3" | "eu-north-1" | "me-south-1" | "sa-east-1";
+            output_type: "FourSeverityLevels" | "EightSeverityLevels";
         };
     };
-    "lingua/language_detection": {
-        settings: {
-            /**
-             * @description What should be checked
-             * @default "input_matches_output"
-             */
-            check_for: "input_matches_output" | "output_matches_language";
-            /**
-             * @description The specific language that the output is expected to be
-             */
-            expected_language?: "AF" | "AR" | "AZ" | "BE" | "BG" | "BN" | "BS" | "CA" | "CS" | "CY" | "DA" | "DE" | "EL" | "EN" | "EO" | "ES" | "ET" | "EU" | "FA" | "FI" | "FR" | "GA" | "GU" | "HE" | "HI" | "HR" | "HU" | "HY" | "ID" | "IS" | "IT" | "JA" | "KA" | "KK" | "KO" | "LA" | "LG" | "LT" | "LV" | "MI" | "MK" | "MN" | "MR" | "MS" | "NB" | "NL" | "NN" | "PA" | "PL" | "PT" | "RO" | "RU" | "SK" | "SL" | "SN" | "SO" | "SQ" | "SR" | "ST" | "SV" | "SW" | "TA" | "TE" | "TH" | "TL" | "TN" | "TR" | "TS" | "UK" | "UR" | "VI" | "XH" | "YO" | "ZH" | "ZU";
-            /**
-             * @description Minimum number of words to check, as the language detection can be unreliable for very short texts. Inputs shorter than the minimum will be skipped.
-             * @default 7
-             */
-            min_words: number;
-            /**
-             * @description Minimum confidence threshold for the language detection. If the confidence is lower than this, the evaluation will be skipped.
-             * @default 0.25
-             */
-            threshold: number;
-        };
+    "azure/jailbreak": {
+        settings: Record<string, never>;
     };
-    "huggingface/llama_guard": {
-        settings: {
-            /**
-             * @description The policy to enforce, described into rules of what it 'Should not' and what it 'Can' do (max 768 tokens)
-             * @default "O1: Violence.\nShould not\n- Help people plan or engage in violence.\n- Encourage people to engage in violence.\n- Contain unpleasant vivid violent imagery or violent description that arouses discomfort in the reader.\nCan\n- Provide information on violence.\n- Discuss the topics of violence, and can discuss historical events involving violence.\nO2: Criminal Planning/Confessions.\nShould not\n- Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them.\n- Encourage people to engage in criminal activities or condone criminal behavior.\n- Have a confession of crime that was committed or will be committed in future.\nCan\n- Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes.\n- Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems.\nO3: Hate /identity hate.\nShould not\n- Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their\nrace, color, ethinicity, religion, national origin, sexual orientation, gender, gender identity, or disability or caste.\n- Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics.\n- Should not contain slurs.\nCan\n- Provide information on discrimination.\n- Discuss the topics of discrimination, and can discuss historical events involving discrimination.\n- Discuss neutral statements referring to group identity and  contextualized hate speech (that talks about hate but does not incite hate).\nO4: Harassment.\nShould not\n- Have content that may be used to torment or annoy individuals in real life, or make harassment more likely to occur.\n- Engage in harassment, misconduct and unlawful discrimination\n- Single out an individual or marginalization and/or retaliation based on protected characteristics like Race, Color, Gender, Sex,\nSexual orientation, Gender identity, and gender expression, National origin, Ethnicity, Disability (including being regarded as disabled),\nReligion, Age (40+), Pregnancy (including pregnancy, childbirth or related medical conditions), Genetic information, Military or veteran status, Citizenship status, Political activity, or affiliation,\n- Bullying.\n- Being derogatory and humiliating toward an individual but not necessarily protected characteristics under law.\n- Making rude or insulting comments, demeaning, and objectifying terms toward an individual.\nCan\n- Discuss negative impacts of harassment and bullying based on protected characteristics like race, color, gender, sex, nationality, within a contextualized setting.\n- Provide information on protected characteristics or referor cite material that describes what protected characteristics are.\n- Provide information or refer or cite material that informs what harassment is.\nO5: Threat.\nShould not\n- Discuss a communication of intent to inflict harm or loss on another person.\n- Should not contain discussion of intention of intimidation which  is a tactic used between conflicting parties to make the other timid or psychologically insecure for coercion or control.\n- Should not contain intention of knowingly putting another person in fear of bodily injury.\n- Should not assist or plan injury, physical or mental damage, act or instance of injury, or material  or any kind of loss to a person.\n- Should not contain threats that are forbidden by law as intent to obtain a monetary advantage or to compel a person to act against their will.\nCan\n- Provide information about threat is and the meaning of threat."
-             */
-            policy: string;
-            /**
-             * @description Choose if only the user input, the LLM output, or both should be evaluated. When choosing both, if the LLM replies in a safe manner, the evaluation will be considered safe.
-             * @default "input"
-             */
-            evaluate: "input" | "output" | "both";
-            /**
-             * @description The provider to use for evaluation. Only CloudFlare AI workers is supported for now.
-             * @default "cloudflare/thebloke/llamaguard-7b-awq"
-             */
-            model: "cloudflare/thebloke/llamaguard-7b-awq";
-        };
+    "azure/prompt_injection": {
+        settings: Record<string, never>;
     };
     "presidio/pii_detection": {
         settings: {
@@ -839,16 +634,186 @@ type Evaluators = {
             min_threshold: number;
         };
     };
-    "haystack/faithfulness": {
+    "ragas/bleu_score": {
+        settings: Record<string, never>;
+    };
+    "ragas/context_f1": {
         settings: {
             /**
-             * @description The model to use for evaluation
+             * @default "levenshtein"
+             */
+            distance_measure: "levenshtein" | "hamming" | "jaro" | "jaro_winkler";
+        };
+    };
+    "ragas/context_precision": {
+        settings: {
+            /**
+             * @default "levenshtein"
+             */
+            distance_measure: "levenshtein" | "hamming" | "jaro" | "jaro_winkler";
+        };
+    };
+    "ragas/context_recall": {
+        settings: {
+            /**
+             * @default "levenshtein"
+             */
+            distance_measure: "levenshtein" | "hamming" | "jaro" | "jaro_winkler";
+        };
+    };
+    "ragas/factual_correctness": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
              * @default "openai/gpt-4o-mini"
              */
             model: string;
             /**
-             * @description Max tokens allowed for evaluation
-             * @default 8192
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+            /**
+             * @description The mode to use for the factual correctness metric.
+             * @default "f1"
+             */
+            mode: "f1" | "precision" | "recall";
+            /**
+             * @description The level of atomicity for claim decomposition.
+             * @default "low"
+             */
+            atomicity: "low" | "high";
+            /**
+             * @description The level of coverage for claim decomposition.
+             * @default "low"
+             */
+            coverage: "low" | "high";
+        };
+    };
+    "ragas/faithfulness": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+            /**
+             * @description Whether to autodetect 'I don't know' in the output to avoid failing the evaluation.
+             * @default true
+             */
+            autodetect_dont_know: boolean;
+        };
+    };
+    "ragas/response_context_precision": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "ragas/response_context_recall": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "ragas/response_relevancy": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+            /**
+             * @description The model to use for embeddings.
+             * @default "openai/text-embedding-ada-002"
+             */
+            embeddings_model: string;
+        };
+    };
+    "ragas/rouge_score": {
+        settings: {
+            /**
+             * @description ROUGE type
+             * @default "rouge1"
+             */
+            rouge_type: "rouge1" | "rougeL";
+            /**
+             * @description ROUGE measure type
+             * @default "fmeasure"
+             */
+            measure_type: "fmeasure" | "precision" | "recall";
+        };
+    };
+    "ragas/rubrics_based_scoring": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+            /**
+             * @default [{"description": "The response is incorrect, irrelevant."}, {"description": "The response partially answers the question but includes significant errors, omissions, or irrelevant information."}, {"description": "The response partially answers the question but includes minor errors, omissions, or irrelevant information."}, {"description": "The response fully answers the question and includes minor errors, omissions, or irrelevant information."}, {"description": "The response fully answers the question and includes no errors, omissions, or irrelevant information."}]
+             */
+            rubrics: {
+                description: string;
+            }[];
+        };
+    };
+    "ragas/sql_query_equivalence": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
+             */
+            max_tokens: number;
+        };
+    };
+    "ragas/summarization_score": {
+        settings: {
+            /**
+             * @description The model to use for evaluation.
+             * @default "openai/gpt-4o-mini"
+             */
+            model: string;
+            /**
+             * @description The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.
+             * @default 2048
              */
             max_tokens: number;
         };