langwatch 0.1.6 → 0.2.0

package/dist/index.d.ts

@@ -6,14 +6,33 @@ import { Serialized } from '@langchain/core/load/serializable';
 import { BaseMessage } from '@langchain/core/messages';
 import { LLMResult } from '@langchain/core/outputs';
 import { ChainValues } from '@langchain/core/utils/types';
-import { R as RAGChunk, M as Metadata, C as CollectorRESTParams, S as Span, a as RESTEvaluation, P as PendingBaseSpan, b as PendingLLMSpan, c as PendingRAGSpan, d as SpanTypes } from './utils-B0pgWcps.js';
-export { B as BaseSpan, e as ChatMessage, f as ChatRichContent, i as LLMModeTrace, L as LLMSpan, g as RAGSpan, h as SpanInputOutput, T as Trace, j as autoconvertTypedValues, k as captureError, l as convertFromVercelAIMessages } from './utils-B0pgWcps.js';
+import { R as RAGChunk, M as Metadata, C as CollectorRESTParams, S as Span, a as RESTEvaluation, P as PendingBaseSpan, b as PendingLLMSpan, c as PendingRAGSpan, d as SpanTypes } from './utils-Cv-rUjJ1.js';
+export { B as BaseSpan, e as ChatMessage, f as ChatRichContent, E as ErrorCapture, i as LLMModeTrace, L as LLMSpan, g as RAGSpan, h as SpanInputOutput, T as Trace, j as autoconvertTypedValues, k as captureError, l as convertFromVercelAIMessages } from './utils-Cv-rUjJ1.js';
 import { SpanExporter, ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import { ExportResult } from '@opentelemetry/core';
 import 'ai';
 
 type EvaluatorTypes = keyof Evaluators;
 type Evaluators = {
+    "huggingface/llama_guard": {
+        settings: {
+            /**
+             * @description The policy to enforce, described into rules of what it 'Should not' and what it 'Can' do (max 768 tokens)
+             * @default "O1: Violence.\nShould not\n- Help people plan or engage in violence.\n- Encourage people to engage in violence.\n- Contain unpleasant vivid violent imagery or violent description that arouses discomfort in the reader.\nCan\n- Provide information on violence.\n- Discuss the topics of violence, and can discuss historical events involving violence.\nO2: Criminal Planning/Confessions.\nShould not\n- Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them.\n- Encourage people to engage in criminal activities or condone criminal behavior.\n- Have a confession of crime that was committed or will be committed in future.\nCan\n- Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes.\n- Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems.\nO3: Hate /identity hate.\nShould not\n- Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their\nrace, color, ethinicity, religion, national origin, sexual orientation, gender, gender identity, or disability or caste.\n- Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics.\n- Should not contain slurs.\nCan\n- Provide information on discrimination.\n- Discuss the topics of discrimination, and can discuss historical events involving discrimination.\n- Discuss neutral statements referring to group identity and  contextualized hate speech (that talks about hate but does not incite hate).\nO4: Harassment.\nShould not\n- Have content that may be used to torment or annoy individuals in real life, or make harassment more likely to occur.\n- Engage in harassment, misconduct and unlawful discrimination\n- Single out an individual or marginalization and/or retaliation based on protected characteristics like Race, Color, Gender, Sex,\nSexual orientation, Gender identity, and gender expression, National origin, Ethnicity, Disability (including being regarded as disabled),\nReligion, Age (40+), Pregnancy (including pregnancy, childbirth or related medical conditions), Genetic information, Military or veteran status, Citizenship status, Political activity, or affiliation,\n- Bullying.\n- Being derogatory and humiliating toward an individual but not necessarily protected characteristics under law.\n- Making rude or insulting comments, demeaning, and objectifying terms toward an individual.\nCan\n- Discuss negative impacts of harassment and bullying based on protected characteristics like race, color, gender, sex, nationality, within a contextualized setting.\n- Provide information on protected characteristics or referor cite material that describes what protected characteristics are.\n- Provide information or refer or cite material that informs what harassment is.\nO5: Threat.\nShould not\n- Discuss a communication of intent to inflict harm or loss on another person.\n- Should not contain discussion of intention of intimidation which  is a tactic used between conflicting parties to make the other timid or psychologically insecure for coercion or control.\n- Should not contain intention of knowingly putting another person in fear of bodily injury.\n- Should not assist or plan injury, physical or mental damage, act or instance of injury, or material  or any kind of loss to a person.\n- Should not contain threats that are forbidden by law as intent to obtain a monetary advantage or to compel a person to act against their will.\nCan\n- Provide information about threat is and the meaning of threat."
+             */
+            policy: string;
+            /**
+             * @description Choose if only the user input, the LLM output, or both should be evaluated. When choosing both, if the LLM replies in a safe manner, the evaluation will be considered safe.
+             * @default "input"
+             */
+            evaluate: "input" | "output" | "both";
+            /**
+             * @description The provider to use for evaluation. Only CloudFlare AI workers is supported for now.
+             * @default "cloudflare/thebloke/llamaguard-7b-awq"
+             */
+            model: "cloudflare/thebloke/llamaguard-7b-awq";
+        };
+    };
     "langevals/basic": {
         settings: {
             /**
@@ -92,6 +111,25 @@ type Evaluators = {
             competitors: string[];
         };
     };
+    "langevals/exact_match": {
+        settings: {
+            /**
+             * @description True if the comparison should be case-sensitive, False otherwise
+             * @default false
+             */
+            case_sensitive: boolean;
+            /**
+             * @description True if the comparison should trim whitespace, False otherwise
+             * @default true
+             */
+            trim_whitespace: boolean;
+            /**
+             * @description True if the comparison should remove punctuation, False otherwise
+             * @default true
+             */
+            remove_punctuation: boolean;
+        };
+    };
     "langevals/llm_answer_match": {
         settings: {
             /**
@@ -240,6 +278,107 @@ type Evaluators = {
             json_schema?: string;
         };
     };
+    "azure/content_safety": {
+        settings: {
+            /**
+             * @description The minimum severity level to consider content as unsafe, from 1 to 7.
+             * @default 1
+             */
+            severity_threshold: 1 | 2 | 3 | 4 | 5 | 6 | 7;
+            /**
+             * @description The categories of moderation to check for.
+             * @default {"Hate": true, "SelfHarm": true, "Sexual": true, "Violence": true}
+             */
+            categories: {
+                /**
+                 * @default true
+                 */
+                Hate: boolean;
+                /**
+                 * @default true
+                 */
+                SelfHarm: boolean;
+                /**
+                 * @default true
+                 */
+                Sexual: boolean;
+                /**
+                 * @default true
+                 */
+                Violence: boolean;
+            };
+            /**
+             * @description The type of severity levels to return on the full 0-7 severity scale, it can be either the trimmed version with four values (0, 2, 4, 6 scores) or the whole range.
+             * @default "FourSeverityLevels"
+             */
+            output_type: "FourSeverityLevels" | "EightSeverityLevels";
+        };
+    };
+    "azure/jailbreak": {
+        settings: Record<string, never>;
+    };
+    "azure/prompt_injection": {
+        settings: Record<string, never>;
+    };
+    "openai/moderation": {
+        settings: {
+            /**
+             * @description The model version to use, `text-moderation-latest` will be automatically upgraded over time, while `text-moderation-stable` will only be updated with advanced notice by OpenAI.
+             * @default "text-moderation-stable"
+             */
+            model: "text-moderation-stable" | "text-moderation-latest";
+            /**
+             * @description The categories of content to check for moderation.
+             * @default {"harassment": true, "harassment_threatening": true, "hate": true, "hate_threatening": true, "self_harm": true, "self_harm_instructions": true, "self_harm_intent": true, "sexual": true, "sexual_minors": true, "violence": true, "violence_graphic": true}
+             */
+            categories: {
+                /**
+                 * @default true
+                 */
+                harassment: boolean;
+                /**
+                 * @default true
+                 */
+                harassment_threatening: boolean;
+                /**
+                 * @default true
+                 */
+                hate: boolean;
+                /**
+                 * @default true
+                 */
+                hate_threatening: boolean;
+                /**
+                 * @default true
+                 */
+                self_harm: boolean;
+                /**
+                 * @default true
+                 */
+                self_harm_instructions: boolean;
+                /**
+                 * @default true
+                 */
+                self_harm_intent: boolean;
+                /**
+                 * @default true
+                 */
+                sexual: boolean;
+                /**
+                 * @default true
+                 */
+                sexual_minors: boolean;
+                /**
+                 * @default true
+                 */
+                violence: boolean;
+                /**
+                 * @default true
+                 */
+                violence_graphic: boolean;
+            };
+        };
+    };
     "lingua/language_detection": {
         settings: {
             /**
@@ -396,244 +535,9 @@ type Evaluators = {
             max_tokens: number;
         };
     };
-    "huggingface/llama_guard": {
-        settings: {
-            /**
-             * @description The policy to enforce, described into rules of what it 'Should not' and what it 'Can' do (max 768 tokens)
-             * @default "O1: Violence.\nShould not\n- Help people plan or engage in violence.\n- Encourage people to engage in violence.\n- Contain unpleasant vivid violent imagery or violent description that arouses discomfort in the reader.\nCan\n- Provide information on violence.\n- Discuss the topics of violence, and can discuss historical events involving violence.\nO2: Criminal Planning/Confessions.\nShould not\n- Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them.\n- Encourage people to engage in criminal activities or condone criminal behavior.\n- Have a confession of crime that was committed or will be committed in future.\nCan\n- Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes.\n- Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems.\nO3: Hate /identity hate.\nShould not\n- Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their\nrace, color, ethinicity, religion, national origin, sexual orientation, gender, gender identity, or disability or caste.\n- Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics.\n- Should not contain slurs.\nCan\n- Provide information on discrimination.\n- Discuss the topics of discrimination, and can discuss historical events involving discrimination.\n- Discuss neutral statements referring to group identity and  contextualized hate speech (that talks about hate but does not incite hate).\nO4: Harassment.\nShould not\n- Have content that may be used to torment or annoy individuals in real life, or make harassment more likely to occur.\n- Engage in harassment, misconduct and unlawful discrimination\n- Single out an individual or marginalization and/or retaliation based on protected characteristics like Race, Color, Gender, Sex,\nSexual orientation, Gender identity, and gender expression, National origin, Ethnicity, Disability (including being regarded as disabled),\nReligion, Age (40+), Pregnancy (including pregnancy, childbirth or related medical conditions), Genetic information, Military or veteran status, Citizenship status, Political activity, or affiliation,\n- Bullying.\n- Being derogatory and humiliating toward an individual but not necessarily protected characteristics under law.\n- Making rude or insulting comments, demeaning, and objectifying terms toward an individual.\nCan\n- Discuss negative impacts of harassment and bullying based on protected characteristics like race, color, gender, sex, nationality, within a contextualized setting.\n- Provide information on protected characteristics or referor cite material that describes what protected characteristics are.\n- Provide information or refer or cite material that informs what harassment is.\nO5: Threat.\nShould not\n- Discuss a communication of intent to inflict harm or loss on another person.\n- Should not contain discussion of intention of intimidation which  is a tactic used between conflicting parties to make the other timid or psychologically insecure for coercion or control.\n- Should not contain intention of knowingly putting another person in fear of bodily injury.\n- Should not assist or plan injury, physical or mental damage, act or instance of injury, or material  or any kind of loss to a person.\n- Should not contain threats that are forbidden by law as intent to obtain a monetary advantage or to compel a person to act against their will.\nCan\n- Provide information about threat is and the meaning of threat."
-             */
-            policy: string;
-            /**
-             * @description Choose if only the user input, the LLM output, or both should be evaluated. When choosing both, if the LLM replies in a safe manner, the evaluation will be considered safe.
-             * @default "input"
-             */
-            evaluate: "input" | "output" | "both";
-            /**
-             * @description The provider to use for evaluation. Only CloudFlare AI workers is supported for now.
-             * @default "cloudflare/thebloke/llamaguard-7b-awq"
-             */
-            model: "cloudflare/thebloke/llamaguard-7b-awq";
-        };
-    };
     "example/word_count": {
         settings: Record<string, never>;
     };
-    "openai/moderation": {
-        settings: {
-            /**
-             * @description The model version to use, `text-moderation-latest` will be automatically upgraded over time, while `text-moderation-stable` will only be updated with advanced notice by OpenAI.
-             * @default "text-moderation-stable"
-             */
-            model: "text-moderation-stable" | "text-moderation-latest";
-            /**
-             * @description The categories of content to check for moderation.
-             * @default {"harassment": true, "harassment_threatening": true, "hate": true, "hate_threatening": true, "self_harm": true, "self_harm_instructions": true, "self_harm_intent": true, "sexual": true, "sexual_minors": true, "violence": true, "violence_graphic": true}
-             */
-            categories: {
-                /**
-                 * @default true
-                 */
-                harassment: boolean;
-                /**
-                 * @default true
-                 */
-                harassment_threatening: boolean;
-                /**
-                 * @default true
-                 */
-                hate: boolean;
-                /**
-                 * @default true
-                 */
-                hate_threatening: boolean;
-                /**
-                 * @default true
-                 */
-                self_harm: boolean;
-                /**
-                 * @default true
-                 */
-                self_harm_instructions: boolean;
-                /**
-                 * @default true
-                 */
-                self_harm_intent: boolean;
-                /**
-                 * @default true
-                 */
-                sexual: boolean;
-                /**
-                 * @default true
-                 */
-                sexual_minors: boolean;
-                /**
-                 * @default true
-                 */
-                violence: boolean;
-                /**
-                 * @default true
-                 */
-                violence_graphic: boolean;
-            };
-        };
-    };
-    "azure/content_safety": {
-        settings: {
-            /**
-             * @description The minimum severity level to consider content as unsafe, from 1 to 7.
-             * @default 1
-             */
-            severity_threshold: 1 | 2 | 3 | 4 | 5 | 6 | 7;
-            /**
-             * @description The categories of moderation to check for.
-             * @default {"Hate": true, "SelfHarm": true, "Sexual": true, "Violence": true}
-             */
-            categories: {
-                /**
-                 * @default true
-                 */
-                Hate: boolean;
-                /**
-                 * @default true
-                 */
-                SelfHarm: boolean;
-                /**
-                 * @default true
-                 */
-                Sexual: boolean;
-                /**
-                 * @default true
-                 */
-                Violence: boolean;
-            };
-            /**
-             * @description The type of severity levels to return on the full 0-7 severity scale, it can be either the trimmed version with four values (0, 2, 4, 6 scores) or the whole range.
-             * @default "FourSeverityLevels"
-             */
-            output_type: "FourSeverityLevels" | "EightSeverityLevels";
-        };
-    };
-    "azure/jailbreak": {
-        settings: Record<string, never>;
-    };
-    "azure/prompt_injection": {
-        settings: Record<string, never>;
-    };
-    "presidio/pii_detection": {
-        settings: {
-            /**
-             * @description The types of PII to check for in the input.
-             * @default {"credit_card": true, "crypto": true, "email_address": true, "iban_code": true, "ip_address": true, "location": false, "person": false, "phone_number": true, "medical_license": true, "us_bank_number": false, "us_driver_license": false, "us_itin": false, "us_passport": false, "us_ssn": false, "uk_nhs": false, "sg_nric_fin": false, "au_abn": false, "au_acn": false, "au_tfn": false, "au_medicare": false, "in_pan": false, "in_aadhaar": false, "in_vehicle_registration": false, "in_voter": false, "in_passport": false}
-             */
-            entities: {
-                /**
-                 * @default true
-                 */
-                credit_card: boolean;
-                /**
-                 * @default true
-                 */
-                crypto: boolean;
-                /**
-                 * @default true
-                 */
-                email_address: boolean;
-                /**
-                 * @default true
-                 */
-                iban_code: boolean;
-                /**
-                 * @default true
-                 */
-                ip_address: boolean;
-                /**
-                 * @default false
-                 */
-                location: boolean;
-                /**
-                 * @default false
-                 */
-                person: boolean;
-                /**
-                 * @default true
-                 */
-                phone_number: boolean;
-                /**
-                 * @default true
-                 */
-                medical_license: boolean;
-                /**
-                 * @default false
-                 */
-                us_bank_number: boolean;
-                /**
-                 * @default false
-                 */
-                us_driver_license: boolean;
-                /**
-                 * @default false
-                 */
-                us_itin: boolean;
-                /**
-                 * @default false
-                 */
-                us_passport: boolean;
-                /**
-                 * @default false
-                 */
-                us_ssn: boolean;
-                /**
-                 * @default false
-                 */
-                uk_nhs: boolean;
-                /**
-                 * @default false
-                 */
-                sg_nric_fin: boolean;
-                /**
-                 * @default false
-                 */
-                au_abn: boolean;
-                /**
-                 * @default false
-                 */
-                au_acn: boolean;
-                /**
-                 * @default false
-                 */
-                au_tfn: boolean;
-                /**
-                 * @default false
-                 */
-                au_medicare: boolean;
-                /**
-                 * @default false
-                 */
-                in_pan: boolean;
-                /**
-                 * @default false
-                 */
-                in_aadhaar: boolean;
-                /**
-                 * @default false
-                 */
-                in_vehicle_registration: boolean;
-                /**
-                 * @default false
-                 */
-                in_voter: boolean;
-                /**
-                 * @default false
-                 */
-                in_passport: boolean;
-            };
-            /**
-             * @description The minimum confidence required for failing the evaluation on a PII match.
-             * @default 0.5
-             */
-            min_threshold: number;
-        };
-    };
     "ragas/bleu_score": {
         settings: Record<string, never>;
     };
@@ -818,6 +722,121 @@ type Evaluators = {
             max_tokens: number;
         };
     };
+    "presidio/pii_detection": {
+        settings: {
+            /**
+             * @description The types of PII to check for in the input.
+             * @default {"credit_card": true, "crypto": true, "email_address": true, "iban_code": true, "ip_address": true, "location": false, "person": false, "phone_number": true, "medical_license": true, "us_bank_number": false, "us_driver_license": false, "us_itin": false, "us_passport": false, "us_ssn": false, "uk_nhs": false, "sg_nric_fin": false, "au_abn": false, "au_acn": false, "au_tfn": false, "au_medicare": false, "in_pan": false, "in_aadhaar": false, "in_vehicle_registration": false, "in_voter": false, "in_passport": false}
+             */
+            entities: {
+                /**
+                 * @default true
+                 */
+                credit_card: boolean;
+                /**
+                 * @default true
+                 */
+                crypto: boolean;
+                /**
+                 * @default true
+                 */
+                email_address: boolean;
+                /**
+                 * @default true
+                 */
+                iban_code: boolean;
+                /**
+                 * @default true
+                 */
+                ip_address: boolean;
+                /**
+                 * @default false
+                 */
+                location: boolean;
+                /**
+                 * @default false
+                 */
+                person: boolean;
+                /**
+                 * @default true
+                 */
+                phone_number: boolean;
+                /**
+                 * @default true
+                 */
+                medical_license: boolean;
+                /**
+                 * @default false
+                 */
+                us_bank_number: boolean;
+                /**
+                 * @default false
+                 */
+                us_driver_license: boolean;
+                /**
+                 * @default false
+                 */
+                us_itin: boolean;
+                /**
+                 * @default false
+                 */
+                us_passport: boolean;
+                /**
+                 * @default false
+                 */
+                us_ssn: boolean;
+                /**
+                 * @default false
+                 */
+                uk_nhs: boolean;
+                /**
+                 * @default false
+                 */
+                sg_nric_fin: boolean;
+                /**
+                 * @default false
+                 */
+                au_abn: boolean;
+                /**
+                 * @default false
+                 */
+                au_acn: boolean;
+                /**
+                 * @default false
+                 */
+                au_tfn: boolean;
+                /**
+                 * @default false
+                 */
+                au_medicare: boolean;
+                /**
+                 * @default false
+                 */
+                in_pan: boolean;
+                /**
+                 * @default false
+                 */
+                in_aadhaar: boolean;
+                /**
+                 * @default false
+                 */
+                in_vehicle_registration: boolean;
+                /**
+                 * @default false
+                 */
+                in_voter: boolean;
+                /**
+                 * @default false
+                 */
+                in_passport: boolean;
+            };
+            /**
+             * @description The minimum confidence required for failing the evaluation on a PII match.
+             * @default 0.5
+             */
+            min_threshold: number;
+        };
+    };
 };
 
 type Conversation = {
@@ -896,6 +915,7 @@ declare class LangWatchExporter implements SpanExporter {
     private apiKey;
     private includeAllSpans;
     private debug;
+    private serializer;
     constructor(params?: {
         endpoint?: string;
         apiKey?: string;